diff --git a/Cargo.lock b/Cargo.lock index e216f285b2..5dd5a786c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -91,6 +91,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -257,6 +263,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" @@ -379,7 +391,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -458,6 +470,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.5.0" @@ -769,11 +790,12 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -795,12 +817,30 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "litemap" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -813,6 +853,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.7.5" @@ -832,10 +882,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -848,12 +916,54 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -866,6 +976,35 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -1063,6 +1202,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -1231,6 +1379,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.2.0" @@ -1343,6 +1497,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.11" @@ -1381,6 +1541,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1430,6 +1601,7 @@ dependencies = [ "log", "mime", "once_cell", + "postgres-protocol", "pyo3", "pyo3-log", "pythonize", @@ -1440,6 +1612,7 @@ dependencies = [ "serde_json", "sha2", "tokio", + "tokio-postgres", "ulid", ] @@ -1528,6 +1701,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2 0.6.5", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.2" @@ -1637,12 +1836,33 @@ dependencies = [ "web-time", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "untrusted" version = "0.9.0" @@ -1688,49 +1908,60 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasm-bindgen" -version = "0.2.100" +name = "wasi" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.104", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1738,22 +1969,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.104", - "wasm-bindgen-backend", + "syn 3.0.3", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -1773,9 +2004,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -1791,6 +2022,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1888,6 +2132,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.1" diff --git a/changelog.d/19996.misc b/changelog.d/19996.misc new file mode 100644 index 0000000000..0666dcaaa7 --- /dev/null +++ b/changelog.d/19996.misc @@ -0,0 +1 @@ +Add foundations for a Rust backed database pool and bump minimum supported Rust version to 1.85.0. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2152ad560e..8b40d811f4 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,7 +7,7 @@ name = "synapse" version = "0.1.0" edition = "2021" -rust-version = "1.82.0" +rust-version = "1.85.0" [lib] name = "synapse" @@ -63,8 +63,10 @@ reqwest = { version = "0.12.15", default-features = false, features = [ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } +tokio-postgres = "0.7" once_cell = "1.18.0" itertools = "0.15.0" +postgres-protocol = "0.6.10" [build-dependencies] blake2 = "0.11.0" diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs new file mode 100644 index 0000000000..759966025a --- /dev/null +++ b/rust/src/database/mod.rs @@ -0,0 +1,27 @@ +//! Database access implemented in Rust. +//! +//! Currently this provides a single Postgres backend ([`tokio_postgres`]); see +//! the [`postgres`] submodule. + +pub mod postgres; + +use pyo3::prelude::*; +use pyo3::types::PyModule; + +/// Register the `database` submodule (and its per-backend children) on the +/// top-level `synapse_rust` module. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child = PyModule::new(py, "database")?; + + postgres::register_module(py, &child)?; + + m.add_submodule(&child)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import database` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.database", child)?; + + Ok(()) +} diff --git a/rust/src/database/postgres/helpers.rs b/rust/src/database/postgres/helpers.rs new file mode 100644 index 0000000000..7937daa90a --- /dev/null +++ b/rust/src/database/postgres/helpers.rs @@ -0,0 +1,241 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! Extension traits for driving [`tokio_postgres`] futures to completion from +//! synchronous, GIL-holding Python code. +//! +//! Both [`BlockingPostgres`] and [`BlockingPostgresStream`] release the GIL +//! (`py.detach`) while blocking on the shared tokio runtime, so other Python +//! threads can make progress (and so the runtime's own connection task can run) +//! while we wait. The [`Ungil`] bounds are what let us hand the future across +//! the `detach` boundary. +//! +//! The stream helper is generic over the underlying stream type rather than +//! hard-wired to [`tokio_postgres::RowStream`], so code built on it can be +//! unit-tested against an in-memory stream with no live database — see +//! [`BlockingPostgresStream`]'s tests. + +use std::{ + future::Future, + pin::{pin, Pin}, +}; + +use futures::{stream::FusedStream, FutureExt, StreamExt}; +use pyo3::{marker::Ungil, PyResult, Python}; +use tokio::runtime::Handle; + +use crate::database::postgres::pg_err_to_py; + +/// Block on a future on a given runtime, releasing the GIL while we wait. +/// +/// Note that this drives the future on *this* thread, not on the runtime's +/// worker threads. See [`tokio::runtime::Handle::block_on`] for details. +pub trait BlockingPostgres +where + Self: Future + Sized + Send + Ungil, + Self::Output: Ungil + Send, +{ + /// Drive `self` to completion on the given runtime, returning its output. + /// Releases the GIL for the duration so the wait doesn't block other Python + /// threads. + /// + /// `handle` is expected to be the server's shared runtime (see + /// [`crate::runtime`]). It is passed in explicitly rather than taken + /// from the thread-local context via [`Handle::current`], so callers work + /// from any Python thread with no need to have *entered* the runtime first. + /// The blocking wait runs on the calling (Python) thread, never on a + /// runtime worker, so it cannot starve the worker threads that complete the + /// future. + fn block_on(self, py: Python<'_>, handle: &Handle) -> Self::Output { + // Check if this future is already ready, and if so return immediately + // without needlessly releasing and reacquiring the GIL. + // + // This is basically the same as `FutureExt::now_or_never`, except we a) + // keep the pinned future to poll again below, and b) enter the runtime + // in case the future needs it. + let mut pin_self = pin!(self); + { + let _guard = handle.enter(); + let noop_waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(noop_waker); + match pin_self.poll_unpin(&mut cx) { + std::task::Poll::Ready(val) => return val, + std::task::Poll::Pending => (), + } + } + + // Note: this will drive the future to completion on *this* thread, not + // on the runtime's worker threads. See `Handle::block_on` for details. + py.detach(|| handle.block_on(pin_self)) + } +} + +/// Same as [`BlockingPostgres`], but for futures that yield a +/// [`tokio_postgres::Error`], mapping any error into a Python exception. +pub trait BlockingPostgresResult +where + Self: Future> + Sized + Send + Ungil, + Self::Output: Ungil + Send, +{ + /// Block on `self` on the given runtime and convert a Postgres error into + /// a `PyErr`. + fn block_on_pg_result(self, py: Python<'_>, handle: &Handle) -> PyResult { + self.block_on(py, handle).map_err(pg_err_to_py) + } +} + +// Blanket impls: every suitable future automatically gets `block_on` / +// `block_on_pg_result`, so callers can write `fut.block_on(py, handle)` +// directly. +impl BlockingPostgres for F +where + F: Future + Sized + Send + Ungil, + F::Output: Ungil + Send, +{ +} +impl BlockingPostgresResult for F +where + F: Future> + Sized + Send + Ungil, + F::Output: Ungil + Send, +{ +} + +/// Pull items from a [`FusedStream`] from synchronous Python code, blocking on +/// a given runtime only when the next item isn't already buffered. +/// +/// Implemented for any pinned, fused stream (e.g. `Pin<&mut Fuse>`) whose +/// items can cross the GIL-release boundary, so it can be tested against an +/// in-memory stream as well as a [`tokio_postgres::RowStream`]. +/// +/// The [`FusedStream`] bound provides [`FusedStream::is_terminated`], so +/// [`Self::block_on_next`] can return `None` after exhaustion without polling +/// the stream again. A bare `Stream` is allowed to panic if polled past +/// completion. +pub trait BlockingPostgresStream +where + Self: FusedStream + Sized + Send + Ungil + Unpin, + Self::Item: Ungil + Send, +{ + /// Get the next item from the stream, blocking on the given runtime if + /// necessary. + /// + /// If the stream is not ready to yield an item, this will release the GIL + /// and block until the next item is available. + /// + /// This method will return `None` if the stream is exhausted. + fn block_on_next(&mut self, py: Python<'_>, handle: &Handle) -> Option { + if self.is_terminated() { + return None; + } + + self.next().block_on(py, handle) + } +} + +// Blanket impl over any pinned, fused stream, not just `RowStream`, so the +// tests can use an in-memory stream. +impl BlockingPostgresStream for Pin<&mut S> +where + Self: FusedStream + Send + Ungil + Unpin, + ::Item: Ungil + Send, +{ +} + +#[cfg(test)] +mod tests { + //! These tests don't touch Postgres: the future/stream helpers are generic, + //! so we exercise them with plain async blocks and an in-memory stream. + + use std::pin::pin; + + use futures::stream::{self, StreamExt}; + use tokio::runtime::Runtime; + + use super::*; + + /// A throwaway runtime standing in for the shared one. The helpers take + /// the runtime to block on explicitly, so each test just passes this + /// runtime's handle. + fn test_runtime() -> Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap() + } + + #[test] + fn block_on_runs_future_and_returns_output() { + Python::initialize(); + let rt = test_runtime(); + Python::attach(|py| { + assert_eq!(async { 1 + 2 }.block_on(py, rt.handle()), 3); + }); + } + + #[test] + fn block_on_handles_ready_future() { + Python::initialize(); + let rt = test_runtime(); + Python::attach(|py| { + // A future that returns ready on the first poll. + let fut = std::future::ready(9); + assert_eq!(fut.block_on(py, rt.handle()), 9); + }); + } + + #[test] + fn block_on_handles_not_ready_future() { + Python::initialize(); + let rt = test_runtime(); + Python::attach(|py| { + // A future that requires polling multiple times to complete. + let fut = async { + tokio::task::yield_now().await; + 9 + }; + assert_eq!(fut.block_on(py, rt.handle()), 9); + }); + } + + #[test] + fn block_on_pg_result_maps_ok_through() { + Python::initialize(); + let rt = test_runtime(); + Python::attach(|py| { + let ok = async { Ok::(5) }; + assert_eq!(ok.block_on_pg_result(py, rt.handle()).unwrap(), 5); + // The error path (mapping a `tokio_postgres::Error` to a `PyErr`) + // isn't covered here, because that error type can't be constructed + // by hand. Exercising it needs a live server. + }); + } + + #[test] + fn block_on_next() { + Python::initialize(); + let rt = test_runtime(); + Python::attach(|py| { + // `block_on_next` handles streams that may not be ready on the + // first poll. + let stream = stream::once(async { + tokio::task::yield_now().await; + Ok::(8) + }); + let mut stream = pin!(stream); + assert_eq!(stream.as_mut().block_on_next(py, rt.handle()), Some(Ok(8))); + }); + } +} diff --git a/rust/src/database/postgres/mod.rs b/rust/src/database/postgres/mod.rs new file mode 100644 index 0000000000..dfdfa31532 --- /dev/null +++ b/rust/src/database/postgres/mod.rs @@ -0,0 +1,36 @@ +//! [`tokio_postgres`]-backed Postgres backend for the Rust `database` module. +//! +//! The driver is async. [`helpers`] drives its futures to completion from +//! synchronous Python code on the shared tokio runtime, and [`value`] converts +//! between Python objects and Postgres' binary wire format. + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use pyo3::types::PyModule; + +// `pub` so the items in these submodules count as public API even though +// nothing consumes them yet, which keeps clippy's `dead_code` lint quiet. +// Tighten to private once the connection/cursor code uses them. +pub mod helpers; +pub mod value; + +/// Register the (currently empty) `postgres` submodule under the parent +/// `database` module. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child = PyModule::new(py, "postgres")?; + + m.add_submodule(&child)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust.database import postgres` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.database.postgres", child)?; + + Ok(()) +} + +/// Map a [`tokio_postgres`] error into a Python `RuntimeError`. +fn pg_err_to_py(e: tokio_postgres::Error) -> PyErr { + PyRuntimeError::new_err(format!("postgres error: {e}")) +} diff --git a/rust/src/database/postgres/value.rs b/rust/src/database/postgres/value.rs new file mode 100644 index 0000000000..97c85f8583 --- /dev/null +++ b/rust/src/database/postgres/value.rs @@ -0,0 +1,618 @@ +//! Conversions between Python values and the Postgres SQL value +//! representations. +//! +//! Supports int / float / bool / str / bytes / None. Lists (for +//! `ANY($1)`-style queries) and richer types such as json, decimal and +//! timestamps are not yet supported. +//! +//! The mapping is column-type-driven on the way *out* (a single Python `int` +//! becomes `INT2`/`INT4`/`INT8` depending on the column it is bound to) and +//! type-driven on the way *in*. The supported correspondence is: +//! +//! | [`PgValue`] variant | Python type | Postgres column type(s) | +//! |---------------------|-------------|-------------------------------------| +//! | `Null` | `None` | any (encoded as SQL `NULL`) | +//! | `Bool` | `bool` | `BOOL` | +//! | `Int` | `int` | `INT2`, `INT4`, `INT8` | +//! | `Float` | `float` | `FLOAT4`, `FLOAT8` | +//! | `Text` | `str` | `TEXT`, `VARCHAR`, `NAME`, `BPCHAR` | +//! | `Bytea` | `bytes` | `BYTEA` | +//! +//! Both directions share these type lists via the two `accepts` methods, which +//! must stay in sync with the `match` arms below. + +use std::error::Error; + +use bytes::BytesMut; +use postgres_protocol::types::{ + bool_to_sql, bytea_to_sql, float4_to_sql, float8_to_sql, int2_to_sql, int4_to_sql, int8_to_sql, + text_to_sql, +}; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::pybacked::{PyBackedBytes, PyBackedStr}; +use pyo3::types::{PyBool, PyByteArray, PyBytes, PyFloat, PyInt, PyString, PyTuple}; +use pyo3::{prelude::*, BoundObject}; +use tokio_postgres::types::{to_sql_checked, IsNull, ToSql, Type, WrongType}; + +/// Owned representation of a Python value that we can hand to [`tokio_postgres`] +/// as a [`ToSql`] parameter. +#[derive(Debug)] +pub enum PgValue { + Null, + Bool(bool), + Int(i64), + Float(f64), + Text(PyBackedStr), + Bytea(PyBackedBytes), +} + +impl PgValue { + /// Classify a Python object into a [`PgValue`], or error if its type isn't + /// one we know how to send to Postgres. + /// + /// * `bool` is classified as [`PgValue::Bool`], never [`PgValue::Int`], + /// even though Python's `bool` is a subclass of `int`. + /// * `int` must fit in an `i64`; a larger Python integer raises an + /// `OverflowError` here, since Postgres has no wider integer type in + /// this mapping. + /// * A type we don't recognise raises `TypeError`. + pub fn from_py(obj: Bound) -> PyResult { + if obj.is_none() { + return Ok(PgValue::Null); + } + // `bool` must be checked before `int`, since in Python `bool` is a + // subclass of `int` and would otherwise be caught by the `PyInt` arm. + if let Ok(b) = obj.cast::() { + return Ok(PgValue::Bool(b.is_true())); + } + if let Ok(i) = obj.cast::() { + return Ok(PgValue::Int(i.extract::()?)); + } + if let Ok(f) = obj.cast::() { + return Ok(PgValue::Float(f.value())); + } + + // The `cast_into` calls below are a bit awkward because we need to take + // ownership of `obj`, so we need to use `cast_into` and then + // `into_inner` to get the original `obj` back if the cast fails. + let obj = match obj.cast_into::() { + Ok(s) => return Ok(PgValue::Text(s.try_into()?)), + Err(err) => err.into_inner(), + }; + + let obj = match obj.cast_into::() { + Ok(s) => return Ok(PgValue::Bytea(s.into())), + Err(err) => err.into_inner(), + }; + + let obj = match obj.cast_into::() { + // Annoyingly, PyO3 needs to copy the contents of the byte array + // into a Vec here, so this isn't as cheap as using `PyBytes`. + Ok(s) => return Ok(PgValue::Bytea(s.into())), + Err(err) => err.into_inner(), + }; + + Err(PyTypeError::new_err(format!( + "unsupported parameter type for postgres: {}", + obj.get_type().name()?, + ))) + } +} + +// Lets PyO3 extract a `PgValue` directly from a Python argument, e.g. from a +// method that takes `Option>` as its parameter list. +impl<'a, 'py> FromPyObject<'a, 'py> for PgValue { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + PgValue::from_py(obj.into_bound()) + } +} + +/// Serialises a [`PgValue`] into Postgres' binary wire format. +/// +/// The target column type (`ty`) is supplied by [`tokio_postgres`] from the +/// prepared statement, so the same `PgValue` (e.g. an `Int`) is encoded +/// differently depending on whether the column is `INT2`/`INT4`/`INT8`. A +/// value that doesn't match the column type yields a [`WrongType`] error. +impl ToSql for PgValue { + fn to_sql( + &self, + ty: &Type, + buf: &mut BytesMut, + ) -> Result> { + match (self, ty) { + (PgValue::Null, _) => Ok(IsNull::Yes), + (&PgValue::Bool(v), &Type::BOOL) => { + bool_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT2) => { + let v = i + .try_into() + .map_err(|_| format!("integer {i} out of range for INT2"))?; + int2_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT4) => { + let v = i + .try_into() + .map_err(|_| format!("integer {i} out of range for INT4"))?; + int4_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT8) => { + int8_to_sql(i, buf); + Ok(IsNull::No) + } + (&PgValue::Float(v), &Type::FLOAT4) => { + // The `as` cast narrows to the nearest f32, losing precision. + // Unlike the integer arms there is no fallible conversion to + // use, since floats have no notion of "out of range", just + // varying precision. + float4_to_sql(v as f32, buf); + Ok(IsNull::No) + } + (&PgValue::Float(v), &Type::FLOAT8) => { + float8_to_sql(v, buf); + Ok(IsNull::No) + } + (PgValue::Text(v), &Type::TEXT | &Type::VARCHAR | &Type::NAME | &Type::BPCHAR) => { + text_to_sql(v, buf); + Ok(IsNull::No) + } + (PgValue::Bytea(v), &Type::BYTEA) => { + bytea_to_sql(v, buf); + Ok(IsNull::No) + } + // If we get here then the caller has passed a value that doesn't + // match the type of the column. + (&PgValue::Bool(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Int(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Float(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Text(_), _) => Err(Box::new(WrongType::new::<&str>(ty.clone()))), + (&PgValue::Bytea(_), _) => Err(Box::new(WrongType::new::<&[u8]>(ty.clone()))), + } + } + + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL + | Type::INT2 + | Type::INT4 + | Type::INT8 + | Type::FLOAT4 + | Type::FLOAT8 + | Type::TEXT + | Type::VARCHAR + | Type::NAME + | Type::BPCHAR + | Type::BYTEA + ) + } + + to_sql_checked!(); +} + +/// Convert a Postgres row into a Python tuple, one element per column. +/// +/// Each column is decoded via [`PythonPgFromSql`], so `NULL` becomes `None` and +/// every other supported type becomes its natural Python equivalent. +/// +/// Raises `ValueError` (including the column index and its Postgres type) if a +/// column can't be decoded — e.g. its type isn't in [`PythonPgFromSql::accepts`] +/// or the wire bytes are malformed (such as non-UTF-8 data in a `TEXT` column). +pub fn pg_row_to_py<'py>( + py: Python<'py>, + row: &tokio_postgres::Row, +) -> PyResult> { + let mut output_row = Vec::with_capacity(row.len()); + for idx in 0..row.len() { + let obj: PythonPgFromSql = row.try_get(idx).map_err(|e| { + PyValueError::new_err(format!( + "failed to decode column {idx} (type {}): {e}", + row.columns()[idx].type_() + )) + })?; + output_row.push(obj.0); + } + + PyTuple::new(py, output_row) +} + +/// A decoded column value, ready to drop into a Python tuple. `None` represents +/// SQL `NULL`; otherwise it holds the corresponding Python object. +/// +/// Decoding attaches to the Python interpreter to build the object. So this +/// must only run on a thread that already holds the GIL, and never on a tokio +/// runtime thread. It is private so that [`pg_row_to_py`], which takes a +/// `Python<'py>` token, is the only way to reach it. +/// +/// Use [`pg_row_to_py`] directly. +struct PythonPgFromSql(pub Option>); + +impl<'a> tokio_postgres::types::FromSql<'a> for PythonPgFromSql { + fn from_sql(ty: &Type, raw: &'a [u8]) -> Result> { + // Decoding builds Python objects, so we need the GIL. `try_get` (our + // only caller) already runs under it, so this attach is cheap. This + // also only runs on the python thread and *not* the tokio runtime's + // worker threads, so we don't need to blocking wait for the GIL. + // + // Ideally we'd assert that we already hold the GIL, but PyO3 doesn't + // expose that. + Python::attach(|py| Self::from_sql_with_py(py, ty, raw)) + } + + fn from_sql_null(_ty: &Type) -> Result> { + Ok(PythonPgFromSql(None)) + } + + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL + | Type::INT2 + | Type::INT4 + | Type::INT8 + | Type::FLOAT4 + | Type::FLOAT8 + | Type::TEXT + | Type::VARCHAR + | Type::NAME + | Type::BPCHAR + | Type::BYTEA + ) + } +} + +impl PythonPgFromSql { + /// Decode a non-NULL column value into the matching Python object, given + /// an already-held GIL token. + fn from_sql_with_py( + py: Python<'_>, + ty: &Type, + raw: &[u8], + ) -> Result> { + let obj = match *ty { + Type::BOOL => { + let b = postgres_protocol::types::bool_from_sql(raw)?; + PyBool::new(py, b).into_any().unbind() + } + Type::INT2 => { + let i = postgres_protocol::types::int2_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::INT4 => { + let i = postgres_protocol::types::int4_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::INT8 => { + let i = postgres_protocol::types::int8_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::FLOAT4 => { + let f = postgres_protocol::types::float4_from_sql(raw)?; + PyFloat::new(py, f.into()).into_any().unbind() + } + Type::FLOAT8 => { + let f = postgres_protocol::types::float8_from_sql(raw)?; + PyFloat::new(py, f).into_any().unbind() + } + Type::TEXT | Type::VARCHAR | Type::NAME | Type::BPCHAR => { + // Note: PyString::from_bytes expects UTF-8, and + // postgres_protocol::types::text_from_sql demonstrates that the + // wire bytes are indeed UTF-8, so this should be safe. + PyString::from_bytes(py, raw)?.into_any().unbind() + } + Type::BYTEA => PyBytes::new(py, raw).into_any().unbind(), + _ => { + // This should never happen, unless the `accepts` method is out + // of sync. + return Err(format!("unsupported column type for postgres: {ty}").into()); + } + }; + + Ok(PythonPgFromSql(Some(obj))) + } +} + +#[cfg(test)] +mod tests { + //! These tests exercise the value mapping in isolation — no live Postgres + //! server is needed. The `to_sql` / `from_sql` halves both operate on raw + //! byte buffers, so we can drive them directly with hand-built buffers and + //! a chosen column [`Type`], and the `from_py` classifier just needs a GIL. + + use pyo3::IntoPyObjectExt; + + use super::*; + + /// Encode `value` for column type `ty`, returning the wire bytes and the + /// `IsNull` flag. Panics on a `ToSql` error so callers can assert on the + /// happy path concisely. + fn encode(value: &PgValue, ty: &Type) -> (Vec, bool) { + let mut buf = BytesMut::new(); + let is_null = value.to_sql(ty, &mut buf).expect("encoding should succeed"); + (buf.to_vec(), matches!(is_null, IsNull::Yes)) + } + + /// Like [`encode`], but surfaces the `ToSql` result so a test can assert on + /// the error path (e.g. an out-of-range integer). + fn encode_result(value: &PgValue, ty: &Type) -> Result<(), Box> { + let mut buf = BytesMut::new(); + value.to_sql(ty, &mut buf).map(|_| ()) + } + + /// Build a [`PgValue::Text`] from a Rust string. The variant is backed by a + /// Python `str`, so this needs an attached interpreter. + fn text(py: Python<'_>, s: &str) -> PgValue { + PgValue::Text(PyString::new(py, s).try_into().unwrap()) + } + + /// Build a [`PgValue::Bytea`] from a Rust byte slice. As with [`text`], the + /// variant is backed by a Python `bytes`. + fn bytea(py: Python<'_>, b: &[u8]) -> PgValue { + PgValue::Bytea(PyBytes::new(py, b).into()) + } + + #[test] + fn from_py_classifies_supported_types() { + Python::initialize(); + Python::attach(|py| { + // `None` -> NULL. + assert!(matches!( + PgValue::from_py(py.None().into_bound(py)).unwrap(), + PgValue::Null + )); + + // `bool` must win over `int` (it is an `int` subclass in Python). + assert!(matches!( + PgValue::from_py(true.into_bound_py_any(py).unwrap()).unwrap(), + PgValue::Bool(true) + )); + + assert!(matches!( + PgValue::from_py(7i64.into_bound_py_any(py).unwrap()).unwrap(), + PgValue::Int(7) + )); + + assert!(matches!( + PgValue::from_py(1.5f64.into_bound_py_any(py).unwrap()).unwrap(), + PgValue::Float(1.5) + )); + + match PgValue::from_py("hello".into_bound_py_any(py).unwrap()).unwrap() { + PgValue::Text(s) => assert_eq!(&*s, "hello"), + other => panic!("expected Text, got {other:?}"), + } + + match PgValue::from_py(PyBytes::new(py, b"\x00\xff").into_any()).unwrap() { + PgValue::Bytea(b) => assert_eq!(&*b, b"\x00\xff"), + other => panic!("expected Bytea, got {other:?}"), + } + }); + } + + #[test] + fn from_py_extracts_via_frompyobject() { + // Check that extracting a `PgValue` from a Python object (the + // `FromPyObject` path) forwards to `from_py`. + Python::initialize(); + Python::attach(|py| { + let obj = 7i64.into_pyobject(py).unwrap().into_any(); + assert!(matches!(obj.extract::().unwrap(), PgValue::Int(7))); + }); + } + + #[test] + fn from_py_rejects_unsupported_type() { + Python::initialize(); + Python::attach(|py| { + // A list is not a scalar we know how to bind. + let list = pyo3::types::PyList::new(py, [1, 2, 3]).unwrap(); + let err = PgValue::from_py(list.into_any()).unwrap_err(); + assert!(err.is_instance_of::(py)); + // The message should name the offending type. + assert!(err.to_string().contains("list"), "got: {err}"); + }); + } + + #[test] + fn to_sql_encodes_each_type_for_its_column() { + // NULL is encoded as "no bytes, IsNull::Yes" regardless of column type. + let (bytes, is_null) = encode(&PgValue::Null, &Type::INT4); + assert!(is_null); + assert!(bytes.is_empty()); + + // Integers are width-specific: the same `Int` encodes to 2/4/8 bytes + // depending on the column type. + assert_eq!(encode(&PgValue::Int(1), &Type::INT2).0, 1i16.to_be_bytes()); + assert_eq!(encode(&PgValue::Int(1), &Type::INT4).0, 1i32.to_be_bytes()); + assert_eq!(encode(&PgValue::Int(1), &Type::INT8).0, 1i64.to_be_bytes()); + + assert_eq!(encode(&PgValue::Bool(true), &Type::BOOL).0, vec![1]); + assert_eq!(encode(&PgValue::Bool(false), &Type::BOOL).0, vec![0]); + assert_eq!( + encode(&PgValue::Float(1.0), &Type::FLOAT8).0, + 1.0f64.to_be_bytes() + ); + Python::initialize(); + Python::attach(|py| { + assert_eq!(encode(&text(py, "hi"), &Type::TEXT).0, b"hi".to_vec()); + assert_eq!(encode(&bytea(py, b"\x01\x02"), &Type::BYTEA).0, vec![1, 2]); + }); + } + + #[test] + fn to_sql_float4_narrows_with_precision_loss() { + // `0.1` is not representable in binary floating point, so the f64 and + // f32 encodings genuinely differ. Encoding to a FLOAT4 column must use + // the (lossy) f32 narrowing, not reinterpret the f64 bytes. + let value = 0.1f64; + assert_eq!( + encode(&PgValue::Float(value), &Type::FLOAT4).0, + (value as f32).to_be_bytes() + ); + // And the bytes are not just a truncation of the f64 encoding. + assert_ne!( + encode(&PgValue::Float(value), &Type::FLOAT4).0, + value.to_be_bytes()[..4].to_vec() + ); + } + + #[test] + fn to_sql_rejects_mismatched_column_type() { + // A value whose type doesn't match the column is a `WrongType` error + // rather than a silent reinterpretation. Hit a few distinct arms. + Python::initialize(); + Python::attach(|py| { + let cases: &[(PgValue, Type)] = &[ + (PgValue::Int(1), Type::TEXT), + (text(py, "x"), Type::INT4), + (bytea(py, b"x"), Type::TEXT), + (PgValue::Bool(true), Type::INT4), + (PgValue::Float(1.0), Type::INT8), + ]; + for (value, ty) in cases { + let mut buf = BytesMut::new(); + assert!( + value.to_sql(ty, &mut buf).is_err(), + "expected {value:?} -> {ty} to be rejected" + ); + } + }); + } + + #[test] + fn to_sql_integer_width_boundaries() { + // Boundary values for each width encode; one step past the boundary is + // rejected by the `try_into` guards (INT8 spans all of i64, so there is + // nothing out of range for it). + assert!(encode_result(&PgValue::Int(i16::MAX as i64), &Type::INT2).is_ok()); + assert!(encode_result(&PgValue::Int(i16::MIN as i64), &Type::INT2).is_ok()); + assert!(encode_result(&PgValue::Int(i16::MAX as i64 + 1), &Type::INT2).is_err()); + assert!(encode_result(&PgValue::Int(i16::MIN as i64 - 1), &Type::INT2).is_err()); + + assert!(encode_result(&PgValue::Int(i32::MAX as i64), &Type::INT4).is_ok()); + assert!(encode_result(&PgValue::Int(i32::MIN as i64), &Type::INT4).is_ok()); + assert!(encode_result(&PgValue::Int(i32::MAX as i64 + 1), &Type::INT4).is_err()); + assert!(encode_result(&PgValue::Int(i32::MIN as i64 - 1), &Type::INT4).is_err()); + + assert_eq!( + encode(&PgValue::Int(i64::MAX), &Type::INT8).0, + i64::MAX.to_be_bytes() + ); + assert_eq!( + encode(&PgValue::Int(i64::MIN), &Type::INT8).0, + i64::MIN.to_be_bytes() + ); + } + + #[test] + fn accepts_lists_match_supported_types() { + // The encode and decode sides must accept exactly the supported column + // types and reject everything else; a drift between the two `accepts` + // lists and the `match` arms would surface here. + for ty in [ + Type::BOOL, + Type::INT2, + Type::INT4, + Type::INT8, + Type::FLOAT4, + Type::FLOAT8, + Type::TEXT, + Type::VARCHAR, + Type::NAME, + Type::BPCHAR, + Type::BYTEA, + ] { + assert!(::accepts(&ty), "ToSql should accept {ty}"); + assert!( + ::accepts(&ty), + "FromSql should accept {ty}" + ); + } + + for ty in [Type::JSON, Type::TIMESTAMPTZ, Type::UUID] { + assert!( + !::accepts(&ty), + "ToSql should reject {ty}" + ); + assert!( + !::accepts(&ty), + "FromSql should reject {ty}" + ); + } + } + + #[test] + fn from_sql_decodes_into_python_objects() { + use tokio_postgres::types::FromSql; + + Python::initialize(); + Python::attach(|py| { + // Each supported type round-trips from its wire bytes to the + // matching Python object. This mirrors the encode test's type list. + let int2 = PythonPgFromSql::from_sql(&Type::INT2, &7i16.to_be_bytes()).unwrap(); + assert_eq!(int2.0.unwrap().extract::(py).unwrap(), 7); + + let int4 = PythonPgFromSql::from_sql(&Type::INT4, &42i32.to_be_bytes()).unwrap(); + assert_eq!(int4.0.unwrap().extract::(py).unwrap(), 42); + + // A wide INT8 that wouldn't fit in INT4, to prove the width is honoured. + let big = i64::MAX - 1; + let int8 = PythonPgFromSql::from_sql(&Type::INT8, &big.to_be_bytes()).unwrap(); + assert_eq!(int8.0.unwrap().extract::(py).unwrap(), big); + + // FLOAT4 decodes via an f32, then widens to a Python float. + let f4 = PythonPgFromSql::from_sql(&Type::FLOAT4, &1.5f32.to_be_bytes()).unwrap(); + assert_eq!(f4.0.unwrap().extract::(py).unwrap(), 1.5); + + let f8 = PythonPgFromSql::from_sql(&Type::FLOAT8, &2.5f64.to_be_bytes()).unwrap(); + assert_eq!(f8.0.unwrap().extract::(py).unwrap(), 2.5); + + let b = PythonPgFromSql::from_sql(&Type::BOOL, &[1]).unwrap(); + assert!(b.0.unwrap().extract::(py).unwrap()); + + let text = PythonPgFromSql::from_sql(&Type::TEXT, b"hi").unwrap(); + assert_eq!(text.0.unwrap().extract::(py).unwrap(), "hi"); + + // BYTEA passes raw bytes through unchanged, including NULs and + // non-UTF-8 data — the natural inverse of the encode test. + let bytea = PythonPgFromSql::from_sql(&Type::BYTEA, b"\x00\xff").unwrap(); + assert_eq!( + bytea.0.unwrap().extract::>(py).unwrap(), + b"\x00\xff" + ); + + // A SQL NULL decodes to `None` via `from_sql_null`. + let null = PythonPgFromSql::from_sql_null(&Type::INT4).unwrap(); + assert!(null.0.is_none()); + }); + } + + #[test] + fn from_sql_rejects_unsupported_column_type() { + use tokio_postgres::types::FromSql; + + // The `_ =>` arm in `from_sql_with_py` guards against `accepts` drifting + // out of sync; decoding an unsupported type is an error, not a panic. + Python::initialize(); + Python::attach(|_py| { + assert!(PythonPgFromSql::from_sql(&Type::JSON, b"{}").is_err()); + }); + } + + #[test] + fn from_sql_rejects_non_utf8_text() { + use tokio_postgres::types::FromSql; + + // TEXT decode goes through `PyString::from_bytes`, which validates + // UTF-8: malformed bytes surface as an error rather than a panic. + Python::initialize(); + Python::attach(|_py| { + assert!(PythonPgFromSql::from_sql(&Type::TEXT, b"\xff\xfe").is_err()); + }); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f3a9cec7e8..dab68e4eb9 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -11,6 +11,7 @@ pub mod acl; pub mod canonical_json; pub mod clock; pub mod config; +pub mod database; pub mod deferred; pub mod duration; pub mod errors; @@ -88,6 +89,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { acl::register_module(py, m)?; logging::context::register_module(py, m)?; deferred::register_module(py, m)?; + database::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; handlers::register_module(py, m)?; diff --git a/synapse/synapse_rust/database/__init__.pyi b/synapse/synapse_rust/database/__init__.pyi new file mode 100644 index 0000000000..a9a240552b --- /dev/null +++ b/synapse/synapse_rust/database/__init__.pyi @@ -0,0 +1,11 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/synapse/synapse_rust/database/postgres.pyi b/synapse/synapse_rust/database/postgres.pyi new file mode 100644 index 0000000000..28a4f693c6 --- /dev/null +++ b/synapse/synapse_rust/database/postgres.pyi @@ -0,0 +1,13 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +# Empty stub file for `synapse::database::postgres` module