From 780beaedb98ecef7f4fbf0790452478674ab1e5f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Jun 2026 19:27:58 +0000 Subject: [PATCH] Add Postgres value mapping for the Rust database backend Introduce the Python<->Postgres value-mapping layer that the forthcoming Rust DBAPI2 backend will build on, plus the module scaffolding to hang it off (`synapse_rust.database.postgres`). `PgValue` is an owned representation of a bound parameter that implements `tokio_postgres::types::ToSql`, encoding into Postgres' binary wire format according to the column type from the prepared statement (so a single Python `int` becomes INT2/INT4/INT8 as appropriate, with range checks). `PythonPgFromSql` is the decode counterpart, turning a column's wire bytes back into the natural Python object (and SQL NULL into `None`). The two `accepts` lists are kept in sync via tests. The `value` submodule is exposed as `pub` for now so its not-yet-consumed public items don't trip clippy's `dead_code` lint; later changes that wire it into the cursor/connection code tighten that back up. Covers int / float / bool / str / bytes / None; lists and richer types (json, decimal, timestamps) are left to a follow-up. The unit tests exercise the whole mapping without a live Postgres server, including the float4 lossy narrowing, integer width boundaries, the WrongType and out-of-range paths, and the unsupported/non-UTF-8 decode errors. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 323 ++++++++++++++-- rust/Cargo.toml | 2 + rust/src/database/mod.rs | 27 ++ rust/src/database/postgres/mod.rs | 37 ++ rust/src/database/postgres/value.rs | 581 ++++++++++++++++++++++++++++ rust/src/lib.rs | 2 + 6 files changed, 936 insertions(+), 36 deletions(-) create mode 100644 rust/src/database/mod.rs create mode 100644 rust/src/database/postgres/mod.rs create mode 100644 rust/src/database/postgres/value.rs diff --git a/Cargo.lock b/Cargo.lock index d9eaed981a..d3cf575c57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -257,6 +263,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" @@ -379,7 +391,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -458,6 +470,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -769,11 +790,12 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -805,12 +827,30 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "litemap" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -823,6 +863,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.7.5" @@ -842,10 +892,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -858,12 +926,54 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -876,6 +986,35 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -1073,6 +1212,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -1241,6 +1389,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.2.0" @@ -1353,6 +1507,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.11" @@ -1391,6 +1551,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1441,6 +1612,7 @@ dependencies = [ "log", "mime", "once_cell", + "postgres-protocol", "pyo3", "pyo3-log", "pythonize", @@ -1451,6 +1623,7 @@ dependencies = [ "serde_json", "sha2", "tokio", + "tokio-postgres", "ulid", ] @@ -1539,6 +1712,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2 0.6.5", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.2" @@ -1648,12 +1847,33 @@ dependencies = [ "web-time", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "untrusted" version = "0.9.0" @@ -1699,49 +1919,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasm-bindgen" -version = "0.2.100" +name = "wasi" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.104", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ - "cfg-if", "js-sys", - "once_cell", + "tokio", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1749,22 +1981,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.104", - "wasm-bindgen-backend", + "syn 3.0.3", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -1784,9 +2016,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -1802,6 +2034,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1899,6 +2144,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3e3369f14b..5e8e5682c5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -63,11 +63,13 @@ reqwest = { version = "0.12.15", default-features = false, features = [ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } +tokio-postgres = "0.7" once_cell = "1.18.0" itertools = "0.15.0" # Used by the `libpq` module to load the system libpq at runtime, so that # libpq is an optional runtime dependency rather than a build-time one. libloading = "0.8" +postgres-protocol = "0.6.10" [build-dependencies] blake2 = "0.11.0" diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs new file mode 100644 index 0000000000..42b4bb437a --- /dev/null +++ b/rust/src/database/mod.rs @@ -0,0 +1,27 @@ +//! DBAPI2-shaped Connection / Cursor types implemented in Rust. +//! +//! Currently this provides a single Postgres backend ([`tokio_postgres`]); see +//! the [`postgres`] submodule. + +pub mod postgres; + +use pyo3::prelude::*; +use pyo3::types::PyModule; + +/// Register the `database` submodule (and its per-backend children) on the +/// top-level `synapse_rust` module. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child = PyModule::new(py, "database")?; + + postgres::register_module(py, &child)?; + + m.add_submodule(&child)?; + + // Mirror the convention used by other rust submodules so + // `from synapse.synapse_rust import database` works. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.database", child)?; + + Ok(()) +} diff --git a/rust/src/database/postgres/mod.rs b/rust/src/database/postgres/mod.rs new file mode 100644 index 0000000000..5ad02434dd --- /dev/null +++ b/rust/src/database/postgres/mod.rs @@ -0,0 +1,37 @@ +//! [`tokio_postgres`]-backed Postgres backend for the Rust `database` module. +//! +//! This module will grow the Python-facing `Connection` / `Cursor` classes and +//! the `connect` factory; for now it hosts the value-mapping layer ([`value`]) +//! that converts between Python objects and Postgres' binary wire format. +//! +//! The driver itself is async; the eventual `Connection` / `Cursor` types will +//! drive it from sync Python methods via a shared multi-thread tokio runtime. + +use pyo3::prelude::*; +use pyo3::types::PyModule; + +// `pub` (rather than private) so the value-mapping types are reachable from the +// crate root as public API while nothing inside the crate consumes them yet. +// This is what stops clippy's `dead_code` lint from firing on them before the +// cursor/connection code (added in later changes) wires them up; the visibility +// is tightened back to private once that happens. +pub mod value; + +/// Register the `postgres` submodule under the parent `database` module. +/// +/// The `Connection` / `Cursor` classes and the `connect` factory are added in +/// later changes; for now this just creates the (otherwise empty) submodule so +/// the module tree — and the `value` mapping layer hanging off it — exists. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child = PyModule::new(py, "postgres")?; + + m.add_submodule(&child)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust.database import postgres` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.database.postgres", child)?; + + Ok(()) +} diff --git a/rust/src/database/postgres/value.rs b/rust/src/database/postgres/value.rs new file mode 100644 index 0000000000..01f0868e61 --- /dev/null +++ b/rust/src/database/postgres/value.rs @@ -0,0 +1,581 @@ +//! Conversions between Python values and the Postgres SQL value +//! representations. +//! +//! Kept in its own module so the cursor code stays focused on the DBAPI shape +//! rather than the type-mapping table. +//! +//! First cut: int / float / bool / str / bytes / None. Lists (for +//! `ANY($1)`-style queries) and richer types — json, decimal, timestamps — +//! are deferred to a follow-up. +//! +//! The mapping is column-type-driven on the way *out* (a single Python `int` +//! becomes `INT2`/`INT4`/`INT8` depending on the column it is bound to) and +//! type-driven on the way *in*. The supported correspondence is: +//! +//! | [`PgValue`] variant | Python type | Postgres column type(s) | +//! |---------------------|-------------|-------------------------------------| +//! | `Null` | `None` | any (encoded as SQL `NULL`) | +//! | `Bool` | `bool` | `BOOL` | +//! | `Int` | `int` | `INT2`, `INT4`, `INT8` | +//! | `Float` | `float` | `FLOAT4`, `FLOAT8` | +//! | `Text` | `str` | `TEXT`, `VARCHAR`, `NAME`, `BPCHAR` | +//! | `Bytea` | `bytes` | `BYTEA` | +//! +//! Both directions share these type lists via the two `accepts` methods, which +//! must stay in sync with the `match` arms below. + +use std::error::Error; + +use bytes::BytesMut; +use postgres_protocol::types::{ + bool_to_sql, bytea_to_sql, float4_to_sql, float8_to_sql, int2_to_sql, int4_to_sql, int8_to_sql, + text_to_sql, +}; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::types::{PyBool, PyBytes, PyFloat, PyInt, PyString, PyTuple}; +use pyo3::{prelude::*, BoundObject}; +use tokio_postgres::types::{to_sql_checked, IsNull, ToSql, Type, WrongType}; + +/// Owned representation of a Python value that we can hand to [`tokio_postgres`] +/// as a [`ToSql`] parameter. +#[derive(Debug, Clone)] +pub enum PgValue { + Null, + Bool(bool), + Int(i64), + Float(f64), + Text(Box), + Bytea(Box<[u8]>), +} + +impl PgValue { + /// Classify a Python object into a [`PgValue`], or error if its type isn't + /// one we know how to send to Postgres. + /// + /// Two subtleties worth calling out: + /// * `bool` is classified as [`PgValue::Bool`], never [`PgValue::Int`], + /// even though Python's `bool` is a subclass of `int`. + /// * `int` must fit in an `i64`; a larger Python integer raises an + /// `OverflowError` here, since Postgres has no wider integer type in + /// this mapping. + /// + /// A type we don't recognise raises `TypeError`. + pub fn from_py(obj: &Bound) -> PyResult { + if obj.is_none() { + return Ok(PgValue::Null); + } + // `bool` must be checked before `int`, since in Python `bool` is a + // subclass of `int` and would otherwise be caught by the `PyInt` arm. + if let Ok(b) = obj.cast::() { + return Ok(PgValue::Bool(b.is_true())); + } + if let Ok(i) = obj.cast::() { + return Ok(PgValue::Int(i.extract::()?)); + } + if let Ok(f) = obj.cast::() { + return Ok(PgValue::Float(f.value())); + } + if let Ok(s) = obj.cast::() { + return Ok(PgValue::Text(s.to_str()?.into())); + } + if let Ok(b) = obj.cast::() { + return Ok(PgValue::Bytea(b.as_bytes().into())); + } + Err(PyTypeError::new_err(format!( + "unsupported parameter type for postgres: {}", + obj.get_type().name()?, + ))) + } +} + +// Lets PyO3 extract a `PgValue` directly from a Python argument, e.g. when a +// cursor method takes `Option>` for its parameters. +impl<'a, 'py> FromPyObject<'a, 'py> for PgValue { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + PgValue::from_py(&obj) + } +} + +/// Serialises a [`PgValue`] into Postgres' binary wire format. +/// +/// The target column type (`ty`) is supplied by [`tokio_postgres`] from the +/// prepared statement, so the same `PgValue` (e.g. an `Int`) is encoded +/// differently depending on whether the column is `INT2`/`INT4`/`INT8`. A +/// value that doesn't match the column type yields a [`WrongType`] error. +impl ToSql for PgValue { + fn to_sql( + &self, + ty: &Type, + buf: &mut BytesMut, + ) -> Result> { + match (self, ty) { + (PgValue::Null, _) => Ok(IsNull::Yes), + (&PgValue::Bool(v), &Type::BOOL) => { + bool_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT2) => { + let v = i + .try_into() + .map_err(|_| format!("integer {i} out of range for INT2"))?; + int2_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT4) => { + let v = i + .try_into() + .map_err(|_| format!("integer {i} out of range for INT4"))?; + int4_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT8) => { + int8_to_sql(i, buf); + Ok(IsNull::No) + } + (&PgValue::Float(v), &Type::FLOAT4) => { + // The `as` cast here generates the closest f32 to the f64, + // with loss of precision. Since Python floats are variable + // precision anyway, this is the best we can do. + // + // (Crucially, there is no way of doing a "fallible" cast + // here, since unlike integers there is no notion of "out of + // range" for floats, just varying precision.) + float4_to_sql(v as f32, buf); + Ok(IsNull::No) + } + (&PgValue::Float(v), &Type::FLOAT8) => { + float8_to_sql(v, buf); + Ok(IsNull::No) + } + (PgValue::Text(v), &Type::TEXT | &Type::VARCHAR | &Type::NAME | &Type::BPCHAR) => { + text_to_sql(v, buf); + Ok(IsNull::No) + } + (PgValue::Bytea(v), &Type::BYTEA) => { + bytea_to_sql(v, buf); + Ok(IsNull::No) + } + // If we get here then the caller has passed a value that doesn't + // match the type of the column. + (&PgValue::Bool(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Int(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Float(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Text(_), _) => Err(Box::new(WrongType::new::<&str>(ty.clone()))), + (&PgValue::Bytea(_), _) => Err(Box::new(WrongType::new::<&[u8]>(ty.clone()))), + } + } + + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL + | Type::INT2 + | Type::INT4 + | Type::INT8 + | Type::FLOAT4 + | Type::FLOAT8 + | Type::TEXT + | Type::VARCHAR + | Type::NAME + | Type::BPCHAR + | Type::BYTEA + ) + } + + to_sql_checked!(); +} + +/// Convert a Postgres row into a Python tuple, one element per column. +/// +/// Each column is decoded via [`PythonPgFromSql`], so `NULL` becomes `None` and +/// every other supported type becomes its natural Python equivalent. +/// +/// Raises `ValueError` (including the column index and its Postgres type) if a +/// column can't be decoded — e.g. its type isn't in [`PythonPgFromSql::accepts`] +/// or the wire bytes are malformed (such as non-UTF-8 data in a `TEXT` column). +pub fn pg_row_to_py<'py>( + py: Python<'py>, + row: &tokio_postgres::Row, +) -> PyResult> { + let mut output_row = Vec::with_capacity(row.len()); + for idx in 0..row.len() { + let obj: PythonPgFromSql = row.try_get(idx).map_err(|e| { + PyValueError::new_err(format!( + "failed to decode column {idx} (type {}): {e}", + row.columns()[idx].type_() + )) + })?; + output_row.push(obj.0); + } + + PyTuple::new(py, output_row) +} + +/// A decoded column value, ready to drop into a Python tuple. `None` +/// represents SQL `NULL`; otherwise it holds the corresponding Python object. +pub struct PythonPgFromSql(pub Option>); + +impl<'a> tokio_postgres::types::FromSql<'a> for PythonPgFromSql { + fn from_sql(ty: &Type, raw: &'a [u8]) -> Result> { + // Decoding builds Python objects, so we need the GIL. `try_get` (our + // only caller) already runs under it, so this attach is cheap. + Python::attach(|py| Self::from_sql_with_py(py, ty, raw)) + } + + fn from_sql_null(_ty: &Type) -> Result> { + Ok(PythonPgFromSql(None)) + } + + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL + | Type::INT2 + | Type::INT4 + | Type::INT8 + | Type::FLOAT4 + | Type::FLOAT8 + | Type::TEXT + | Type::VARCHAR + | Type::NAME + | Type::BPCHAR + | Type::BYTEA + ) + } +} + +impl PythonPgFromSql { + /// Decode a non-NULL column value into the matching Python object, given + /// an already-held GIL token. + fn from_sql_with_py( + py: Python<'_>, + ty: &Type, + raw: &[u8], + ) -> Result> { + let obj = match *ty { + Type::BOOL => { + let b = postgres_protocol::types::bool_from_sql(raw)?; + PyBool::new(py, b).into_any().unbind() + } + Type::INT2 => { + let i = postgres_protocol::types::int2_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::INT4 => { + let i = postgres_protocol::types::int4_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::INT8 => { + let i = postgres_protocol::types::int8_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::FLOAT4 => { + let f = postgres_protocol::types::float4_from_sql(raw)?; + PyFloat::new(py, f.into()).into_any().unbind() + } + Type::FLOAT8 => { + let f = postgres_protocol::types::float8_from_sql(raw)?; + PyFloat::new(py, f).into_any().unbind() + } + Type::TEXT | Type::VARCHAR | Type::NAME | Type::BPCHAR => { + PyString::from_bytes(py, raw)?.into_any().unbind() + } + Type::BYTEA => PyBytes::new(py, raw).into_any().unbind(), + _ => { + // This should never happen, unless the `accepts` method is out + // of sync. + return Err(format!("unsupported column type for postgres: {ty}").into()); + } + }; + + Ok(PythonPgFromSql(Some(obj))) + } +} + +#[cfg(test)] +mod tests { + //! These tests exercise the value mapping in isolation — no live Postgres + //! server is needed. The `to_sql` / `from_sql` halves both operate on raw + //! byte buffers, so we can drive them directly with hand-built buffers and + //! a chosen column [`Type`], and the `from_py` classifier just needs a GIL. + + use super::*; + + /// Encode `value` for column type `ty`, returning the wire bytes and the + /// `IsNull` flag. Panics on a `ToSql` error so callers can assert on the + /// happy path concisely. + fn encode(value: &PgValue, ty: &Type) -> (Vec, bool) { + let mut buf = BytesMut::new(); + let is_null = value.to_sql(ty, &mut buf).expect("encoding should succeed"); + (buf.to_vec(), matches!(is_null, IsNull::Yes)) + } + + /// Like [`encode`], but surfaces the `ToSql` result so a test can assert on + /// the error path (e.g. an out-of-range integer). + fn encode_result(value: &PgValue, ty: &Type) -> Result<(), Box> { + let mut buf = BytesMut::new(); + value.to_sql(ty, &mut buf).map(|_| ()) + } + + #[test] + fn from_py_classifies_supported_types() { + Python::initialize(); + Python::attach(|py| { + // `None` -> NULL. + assert!(matches!( + PgValue::from_py(&py.None().into_bound(py)).unwrap(), + PgValue::Null + )); + + // `bool` must win over `int` (it is an `int` subclass in Python). + assert!(matches!( + PgValue::from_py(&true.into_pyobject(py).unwrap()).unwrap(), + PgValue::Bool(true) + )); + + assert!(matches!( + PgValue::from_py(&7i64.into_pyobject(py).unwrap()).unwrap(), + PgValue::Int(7) + )); + + assert!(matches!( + PgValue::from_py(&1.5f64.into_pyobject(py).unwrap()).unwrap(), + PgValue::Float(1.5) + )); + + match PgValue::from_py(&"hello".into_pyobject(py).unwrap()).unwrap() { + PgValue::Text(s) => assert_eq!(&*s, "hello"), + other => panic!("expected Text, got {other:?}"), + } + + match PgValue::from_py(&PyBytes::new(py, b"\x00\xff").into_any()).unwrap() { + PgValue::Bytea(b) => assert_eq!(&*b, b"\x00\xff"), + other => panic!("expected Bytea, got {other:?}"), + } + }); + } + + #[test] + fn from_py_extracts_via_frompyobject() { + // The cursor binds parameters by extracting `PgValue` straight off the + // Python argument; check that `FromPyObject` path forwards to `from_py`. + Python::initialize(); + Python::attach(|py| { + let obj = 7i64.into_pyobject(py).unwrap().into_any(); + assert!(matches!(obj.extract::().unwrap(), PgValue::Int(7))); + }); + } + + #[test] + fn from_py_rejects_unsupported_type() { + Python::initialize(); + Python::attach(|py| { + // A list is not a scalar we know how to bind. + let list = pyo3::types::PyList::new(py, [1, 2, 3]).unwrap(); + let err = PgValue::from_py(&list.into_any()).unwrap_err(); + assert!(err.is_instance_of::(py)); + // The message names the offending type, which is the useful part. + assert!(err.to_string().contains("list"), "got: {err}"); + }); + } + + #[test] + fn to_sql_encodes_each_type_for_its_column() { + // NULL is encoded as "no bytes, IsNull::Yes" regardless of column type. + let (bytes, is_null) = encode(&PgValue::Null, &Type::INT4); + assert!(is_null); + assert!(bytes.is_empty()); + + // Integers are width-specific: the same `Int` encodes to 2/4/8 bytes + // depending on the column type. + assert_eq!(encode(&PgValue::Int(1), &Type::INT2).0, 1i16.to_be_bytes()); + assert_eq!(encode(&PgValue::Int(1), &Type::INT4).0, 1i32.to_be_bytes()); + assert_eq!(encode(&PgValue::Int(1), &Type::INT8).0, 1i64.to_be_bytes()); + + assert_eq!(encode(&PgValue::Bool(true), &Type::BOOL).0, vec![1]); + assert_eq!(encode(&PgValue::Bool(false), &Type::BOOL).0, vec![0]); + assert_eq!( + encode(&PgValue::Float(1.0), &Type::FLOAT8).0, + 1.0f64.to_be_bytes() + ); + assert_eq!( + encode(&PgValue::Text("hi".into()), &Type::TEXT).0, + b"hi".to_vec() + ); + assert_eq!( + encode(&PgValue::Bytea(Box::from(&b"\x01\x02"[..])), &Type::BYTEA).0, + vec![1, 2] + ); + } + + #[test] + fn to_sql_float4_narrows_with_precision_loss() { + // `0.1` is not representable in binary floating point, so the f64 and + // f32 encodings genuinely differ. Encoding to a FLOAT4 column must use + // the (lossy) f32 narrowing, not reinterpret the f64 bytes. + let value = 0.1f64; + assert_eq!( + encode(&PgValue::Float(value), &Type::FLOAT4).0, + (value as f32).to_be_bytes() + ); + // And the result is provably narrower than the FLOAT8 encoding. + assert_ne!( + encode(&PgValue::Float(value), &Type::FLOAT4).0, + value.to_be_bytes()[..4].to_vec() + ); + } + + #[test] + fn to_sql_rejects_mismatched_column_type() { + // A value whose type doesn't match the column is a `WrongType` error + // rather than a silent reinterpretation. Hit a few distinct arms. + let cases: &[(PgValue, Type)] = &[ + (PgValue::Int(1), Type::TEXT), + (PgValue::Text("x".into()), Type::INT4), + (PgValue::Bytea(Box::from(&b"x"[..])), Type::TEXT), + (PgValue::Bool(true), Type::INT4), + (PgValue::Float(1.0), Type::INT8), + ]; + for (value, ty) in cases { + let mut buf = BytesMut::new(); + assert!( + value.to_sql(ty, &mut buf).is_err(), + "expected {value:?} -> {ty} to be rejected" + ); + } + } + + #[test] + fn to_sql_integer_width_boundaries() { + // Boundary values for each width encode; one step past the boundary is + // rejected by the `try_into` guards (INT8 spans all of i64, so there is + // nothing out of range for it). + assert!(encode_result(&PgValue::Int(i16::MAX as i64), &Type::INT2).is_ok()); + assert!(encode_result(&PgValue::Int(i16::MIN as i64), &Type::INT2).is_ok()); + assert!(encode_result(&PgValue::Int(i16::MAX as i64 + 1), &Type::INT2).is_err()); + assert!(encode_result(&PgValue::Int(i16::MIN as i64 - 1), &Type::INT2).is_err()); + + assert!(encode_result(&PgValue::Int(i32::MAX as i64), &Type::INT4).is_ok()); + assert!(encode_result(&PgValue::Int(i32::MIN as i64), &Type::INT4).is_ok()); + assert!(encode_result(&PgValue::Int(i32::MAX as i64 + 1), &Type::INT4).is_err()); + assert!(encode_result(&PgValue::Int(i32::MIN as i64 - 1), &Type::INT4).is_err()); + + assert_eq!( + encode(&PgValue::Int(i64::MAX), &Type::INT8).0, + i64::MAX.to_be_bytes() + ); + assert_eq!( + encode(&PgValue::Int(i64::MIN), &Type::INT8).0, + i64::MIN.to_be_bytes() + ); + } + + #[test] + fn accepts_lists_match_supported_types() { + // The encode and decode sides must accept exactly the supported column + // types and reject everything else; a drift between the two `accepts` + // lists and the `match` arms would surface here. + for ty in [ + Type::BOOL, + Type::INT2, + Type::INT4, + Type::INT8, + Type::FLOAT4, + Type::FLOAT8, + Type::TEXT, + Type::VARCHAR, + Type::NAME, + Type::BPCHAR, + Type::BYTEA, + ] { + assert!(::accepts(&ty), "ToSql should accept {ty}"); + assert!( + ::accepts(&ty), + "FromSql should accept {ty}" + ); + } + + for ty in [Type::JSON, Type::TIMESTAMPTZ, Type::UUID] { + assert!( + !::accepts(&ty), + "ToSql should reject {ty}" + ); + assert!( + !::accepts(&ty), + "FromSql should reject {ty}" + ); + } + } + + #[test] + fn from_sql_decodes_into_python_objects() { + use tokio_postgres::types::FromSql; + + Python::initialize(); + Python::attach(|py| { + // Each supported type round-trips from its wire bytes to the + // matching Python object. This mirrors the encode test's type list. + let int2 = PythonPgFromSql::from_sql(&Type::INT2, &7i16.to_be_bytes()).unwrap(); + assert_eq!(int2.0.unwrap().extract::(py).unwrap(), 7); + + let int4 = PythonPgFromSql::from_sql(&Type::INT4, &42i32.to_be_bytes()).unwrap(); + assert_eq!(int4.0.unwrap().extract::(py).unwrap(), 42); + + // A wide INT8 that wouldn't fit in INT4, to prove the width is honoured. + let big = i64::MAX - 1; + let int8 = PythonPgFromSql::from_sql(&Type::INT8, &big.to_be_bytes()).unwrap(); + assert_eq!(int8.0.unwrap().extract::(py).unwrap(), big); + + // FLOAT4 decodes via an f32, then widens to a Python float. + let f4 = PythonPgFromSql::from_sql(&Type::FLOAT4, &1.5f32.to_be_bytes()).unwrap(); + assert_eq!(f4.0.unwrap().extract::(py).unwrap(), 1.5); + + let f8 = PythonPgFromSql::from_sql(&Type::FLOAT8, &2.5f64.to_be_bytes()).unwrap(); + assert_eq!(f8.0.unwrap().extract::(py).unwrap(), 2.5); + + let b = PythonPgFromSql::from_sql(&Type::BOOL, &[1]).unwrap(); + assert!(b.0.unwrap().extract::(py).unwrap()); + + let text = PythonPgFromSql::from_sql(&Type::TEXT, b"hi").unwrap(); + assert_eq!(text.0.unwrap().extract::(py).unwrap(), "hi"); + + // BYTEA passes raw bytes through unchanged, including NULs and + // non-UTF-8 data — the natural inverse of the encode test. + let bytea = PythonPgFromSql::from_sql(&Type::BYTEA, b"\x00\xff").unwrap(); + assert_eq!( + bytea.0.unwrap().extract::>(py).unwrap(), + b"\x00\xff" + ); + + // A SQL NULL decodes to `None` via `from_sql_null`. + let null = PythonPgFromSql::from_sql_null(&Type::INT4).unwrap(); + assert!(null.0.is_none()); + }); + } + + #[test] + fn from_sql_rejects_unsupported_column_type() { + use tokio_postgres::types::FromSql; + + // The `_ =>` arm in `from_sql_with_py` guards against `accepts` drifting + // out of sync; decoding an unsupported type is an error, not a panic. + Python::initialize(); + Python::attach(|_py| { + assert!(PythonPgFromSql::from_sql(&Type::JSON, b"{}").is_err()); + }); + } + + #[test] + fn from_sql_rejects_non_utf8_text() { + use tokio_postgres::types::FromSql; + + // TEXT decode goes through `PyString::from_bytes`, which validates + // UTF-8: malformed bytes surface as an error rather than a panic. + Python::initialize(); + Python::attach(|_py| { + assert!(PythonPgFromSql::from_sql(&Type::TEXT, b"\xff\xfe").is_err()); + }); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 7374181081..706f5da355 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -11,6 +11,7 @@ pub mod acl; pub mod canonical_json; pub mod clock; pub mod config; +pub mod database; pub mod deferred; pub mod duration; pub mod errors; @@ -89,6 +90,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { acl::register_module(py, m)?; logging::context::register_module(py, m)?; deferred::register_module(py, m)?; + database::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; handlers::register_module(py, m)?;