diff --git a/Cargo.lock b/Cargo.lock index 25ceb4d0c0..eb29a804b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,7 +58,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -70,12 +70,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.0" @@ -103,6 +118,29 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.10.1" @@ -137,6 +175,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -147,17 +194,47 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -181,6 +258,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" @@ -316,11 +399,23 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasi 0.14.2+wasi-0.2.4", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + [[package]] name = "h2" version = "0.4.11" @@ -382,6 +477,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.4.2" @@ -427,6 +531,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.6.0" @@ -710,12 +823,30 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + [[package]] name = "litemap" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -728,6 +859,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.7.5" @@ -763,12 +904,54 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -781,6 +964,35 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.10.1", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -920,7 +1132,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -961,6 +1173,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.4" @@ -968,7 +1186,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -978,7 +1207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", ] [[package]] @@ -990,6 +1219,21 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.12.4" @@ -1158,6 +1402,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.2.0" @@ -1249,8 +1499,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -1260,8 +1510,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -1270,6 +1531,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.11" @@ -1308,6 +1575,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1345,6 +1623,7 @@ dependencies = [ "log", "mime", "once_cell", + "postgres-protocol", "pyo3", "pyo3-log", "pythonize", @@ -1353,8 +1632,9 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tokio", + "tokio-postgres", "ulid", ] @@ -1443,6 +1723,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2 0.6.0", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.2" @@ -1538,9 +1844,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.18.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ulid" @@ -1548,16 +1854,37 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand", + "rand 0.9.4", "web-time", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "untrusted" version = "0.9.0" @@ -1611,6 +1938,15 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.2+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -1715,6 +2051,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" +dependencies = [ + "libredox", + "wasite", + "web-sys", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 612ab09f6d..1d1bee28e4 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -62,8 +62,10 @@ reqwest = { version = "0.12.15", default-features = false, features = [ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } +tokio-postgres = "0.7" once_cell = "1.18.0" itertools = "0.14.0" +postgres-protocol = "0.6.10" [build-dependencies] blake2 = "0.10.4" diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs new file mode 100644 index 0000000000..42b4bb437a --- /dev/null +++ b/rust/src/database/mod.rs @@ -0,0 +1,27 @@ +//! DBAPI2-shaped Connection / Cursor types implemented in Rust. +//! +//! Currently this provides a single Postgres backend ([`tokio_postgres`]); see +//! the [`postgres`] submodule. + +pub mod postgres; + +use pyo3::prelude::*; +use pyo3::types::PyModule; + +/// Register the `database` submodule (and its per-backend children) on the +/// top-level `synapse_rust` module. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child = PyModule::new(py, "database")?; + + postgres::register_module(py, &child)?; + + m.add_submodule(&child)?; + + // Mirror the convention used by other rust submodules so + // `from synapse.synapse_rust import database` works. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.database", child)?; + + Ok(()) +} diff --git a/rust/src/database/postgres/mod.rs b/rust/src/database/postgres/mod.rs new file mode 100644 index 0000000000..5ad02434dd --- /dev/null +++ b/rust/src/database/postgres/mod.rs @@ -0,0 +1,37 @@ +//! [`tokio_postgres`]-backed Postgres backend for the Rust `database` module. +//! +//! This module will grow the Python-facing `Connection` / `Cursor` classes and +//! the `connect` factory; for now it hosts the value-mapping layer ([`value`]) +//! that converts between Python objects and Postgres' binary wire format. +//! +//! The driver itself is async; the eventual `Connection` / `Cursor` types will +//! drive it from sync Python methods via a shared multi-thread tokio runtime. + +use pyo3::prelude::*; +use pyo3::types::PyModule; + +// `pub` (rather than private) so the value-mapping types are reachable from the +// crate root as public API while nothing inside the crate consumes them yet. +// This is what stops clippy's `dead_code` lint from firing on them before the +// cursor/connection code (added in later changes) wires them up; the visibility +// is tightened back to private once that happens. +pub mod value; + +/// Register the `postgres` submodule under the parent `database` module. +/// +/// The `Connection` / `Cursor` classes and the `connect` factory are added in +/// later changes; for now this just creates the (otherwise empty) submodule so +/// the module tree — and the `value` mapping layer hanging off it — exists. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child = PyModule::new(py, "postgres")?; + + m.add_submodule(&child)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust.database import postgres` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.database.postgres", child)?; + + Ok(()) +} diff --git a/rust/src/database/postgres/value.rs b/rust/src/database/postgres/value.rs new file mode 100644 index 0000000000..5bb2c74eb4 --- /dev/null +++ b/rust/src/database/postgres/value.rs @@ -0,0 +1,581 @@ +//! Conversions between Python values and the Postgres SQL value +//! representations. +//! +//! Kept in its own module so the cursor code stays focused on the DBAPI shape +//! rather than the type-mapping table. +//! +//! First cut: int / float / bool / str / bytes / None. Lists (for +//! `ANY($1)`-style queries) and richer types — json, decimal, timestamps — +//! are deferred to a follow-up. +//! +//! The mapping is column-type-driven on the way *out* (a single Python `int` +//! becomes `INT2`/`INT4`/`INT8` depending on the column it is bound to) and +//! type-driven on the way *in*. The supported correspondence is: +//! +//! | [`PgValue`] variant | Python type | Postgres column type(s) | +//! |---------------------|-------------|-------------------------------------| +//! | `Null` | `None` | any (encoded as SQL `NULL`) | +//! | `Bool` | `bool` | `BOOL` | +//! | `Int` | `int` | `INT2`, `INT4`, `INT8` | +//! | `Float` | `float` | `FLOAT4`, `FLOAT8` | +//! | `Text` | `str` | `TEXT`, `VARCHAR`, `NAME`, `BPCHAR` | +//! | `Bytea` | `bytes` | `BYTEA` | +//! +//! Both directions share these type lists via the two `accepts` methods, which +//! must stay in sync with the `match` arms below. + +use std::error::Error; + +use bytes::BytesMut; +use postgres_protocol::types::{ + bool_to_sql, bytea_to_sql, float4_to_sql, float8_to_sql, int2_to_sql, int4_to_sql, int8_to_sql, + text_to_sql, +}; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::types::{PyBool, PyBytes, PyFloat, PyInt, PyString, PyTuple}; +use pyo3::{prelude::*, BoundObject}; +use tokio_postgres::types::{to_sql_checked, IsNull, ToSql, Type, WrongType}; + +/// Owned representation of a Python value that we can hand to [`tokio_postgres`] +/// as a [`ToSql`] parameter. +#[derive(Debug, Clone)] +pub enum PgValue { + Null, + Bool(bool), + Int(i64), + Float(f64), + Text(Box), + Bytea(Box<[u8]>), +} + +impl PgValue { + /// Classify a Python object into a [`PgValue`], or error if its type isn't + /// one we know how to send to Postgres. + /// + /// Two subtleties worth calling out: + /// * `bool` is classified as [`PgValue::Bool`], never [`PgValue::Int`], + /// even though Python's `bool` is a subclass of `int`. + /// * `int` must fit in an `i64`; a larger Python integer raises an + /// `OverflowError` here, since Postgres has no wider integer type in + /// this mapping. + /// + /// A type we don't recognise raises `TypeError`. + pub fn from_py(obj: &Bound) -> PyResult { + if obj.is_none() { + return Ok(PgValue::Null); + } + // `bool` must be checked before `int`, since in Python `bool` is a + // subclass of `int` and would otherwise be caught by the `PyInt` arm. + if let Ok(b) = obj.cast::() { + return Ok(PgValue::Bool(b.is_true())); + } + if let Ok(i) = obj.cast::() { + return Ok(PgValue::Int(i.extract::()?)); + } + if let Ok(f) = obj.cast::() { + return Ok(PgValue::Float(f.value())); + } + if let Ok(s) = obj.cast::() { + return Ok(PgValue::Text(s.to_str()?.into())); + } + if let Ok(b) = obj.cast::() { + return Ok(PgValue::Bytea(b.as_bytes().into())); + } + Err(PyTypeError::new_err(format!( + "unsupported parameter type for postgres: {}", + obj.get_type().name()?, + ))) + } +} + +// Lets PyO3 extract a `PgValue` directly from a Python argument, e.g. when a +// cursor method takes `Option>` for its parameters. +impl<'a, 'py> FromPyObject<'a, 'py> for PgValue { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + PgValue::from_py(&obj) + } +} + +/// Serialises a [`PgValue`] into Postgres' binary wire format. +/// +/// The target column type (`ty`) is supplied by [`tokio_postgres`] from the +/// prepared statement, so the same `PgValue` (e.g. an `Int`) is encoded +/// differently depending on whether the column is `INT2`/`INT4`/`INT8`. A +/// value that doesn't match the column type yields a [`WrongType`] error. +impl ToSql for PgValue { + fn to_sql( + &self, + ty: &Type, + buf: &mut BytesMut, + ) -> Result> { + match (self, ty) { + (PgValue::Null, _) => Ok(IsNull::Yes), + (&PgValue::Bool(v), &Type::BOOL) => { + bool_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT2) => { + let v = i + .try_into() + .map_err(|_| format!("integer {i} out of range for INT2"))?; + int2_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT4) => { + let v = i + .try_into() + .map_err(|_| format!("integer {i} out of range for INT4"))?; + int4_to_sql(v, buf); + Ok(IsNull::No) + } + (&PgValue::Int(i), &Type::INT8) => { + int8_to_sql(i, buf); + Ok(IsNull::No) + } + (&PgValue::Float(v), &Type::FLOAT4) => { + // The `as` cast here generates the closest f32 to the f64, + // with loss of precision. Since Python floats are variable + // precision anyway, this is the best we can do. + // + // (Crucially, there is no way of doing a "fallible" cast + // here, since unlike integers there is no notion of "out of + // range" for floats, just varying precision.) + float4_to_sql(v as f32, buf); + Ok(IsNull::No) + } + (&PgValue::Float(v), &Type::FLOAT8) => { + float8_to_sql(v, buf); + Ok(IsNull::No) + } + (PgValue::Text(v), &Type::TEXT | &Type::VARCHAR | &Type::NAME | &Type::BPCHAR) => { + text_to_sql(v, buf); + Ok(IsNull::No) + } + (PgValue::Bytea(v), &Type::BYTEA) => { + bytea_to_sql(v, buf); + Ok(IsNull::No) + } + // If we get here then the caller has passed a value that doesn't + // match the type of the column. + (&PgValue::Bool(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Int(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Float(_), _) => Err(Box::new(WrongType::new::(ty.clone()))), + (&PgValue::Text(_), _) => Err(Box::new(WrongType::new::<&str>(ty.clone()))), + (&PgValue::Bytea(_), _) => Err(Box::new(WrongType::new::<&[u8]>(ty.clone()))), + } + } + + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL + | Type::INT2 + | Type::INT4 + | Type::INT8 + | Type::FLOAT4 + | Type::FLOAT8 + | Type::TEXT + | Type::VARCHAR + | Type::NAME + | Type::BPCHAR + | Type::BYTEA + ) + } + + to_sql_checked!(); +} + +/// Convert a Postgres row into a Python tuple, one element per column. +/// +/// Each column is decoded via [`PythonPgFromSql`], so `NULL` becomes `None` and +/// every other supported type becomes its natural Python equivalent. +/// +/// Raises `ValueError` (including the column index and its Postgres type) if a +/// column can't be decoded — e.g. its type isn't in [`PythonPgFromSql::accepts`] +/// or the wire bytes are malformed (such as non-UTF-8 data in a `TEXT` column). +pub fn pg_row_to_py<'py>( + py: Python<'py>, + row: &tokio_postgres::Row, +) -> PyResult> { + let mut output_row = Vec::with_capacity(row.len()); + for idx in 0..row.len() { + let obj: PythonPgFromSql = row.try_get(idx).map_err(|e| { + PyValueError::new_err(format!( + "failed to decode column {idx} (type {}): {e}", + row.columns()[idx].type_() + )) + })?; + output_row.push(obj.0); + } + + PyTuple::new(py, output_row) +} + +/// A decoded column value, ready to drop into a Python tuple. `None` +/// represents SQL `NULL`; otherwise it holds the corresponding Python object. +pub struct PythonPgFromSql(pub Option>); + +impl<'a> tokio_postgres::types::FromSql<'a> for PythonPgFromSql { + fn from_sql(ty: &Type, raw: &'a [u8]) -> Result> { + // Decoding builds Python objects, so we need the GIL. `try_get` (our + // only caller) already runs under it, so this attach is cheap. + Python::attach(|py| Self::from_sql_with_py(py, ty, raw)) + } + + fn from_sql_null(_ty: &Type) -> Result> { + Ok(PythonPgFromSql(None)) + } + + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL + | Type::INT2 + | Type::INT4 + | Type::INT8 + | Type::FLOAT4 + | Type::FLOAT8 + | Type::TEXT + | Type::VARCHAR + | Type::NAME + | Type::BPCHAR + | Type::BYTEA + ) + } +} + +impl PythonPgFromSql { + /// Decode a non-NULL column value into the matching Python object, given + /// an already-held GIL token. + fn from_sql_with_py( + py: Python<'_>, + ty: &Type, + raw: &[u8], + ) -> Result> { + let obj = match *ty { + Type::BOOL => { + let b = postgres_protocol::types::bool_from_sql(raw)?; + PyBool::new(py, b).into_any().unbind() + } + Type::INT2 => { + let i = postgres_protocol::types::int2_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::INT4 => { + let i = postgres_protocol::types::int4_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::INT8 => { + let i = postgres_protocol::types::int8_from_sql(raw)?; + PyInt::new(py, i).into_any().unbind() + } + Type::FLOAT4 => { + let f = postgres_protocol::types::float4_from_sql(raw)?; + PyFloat::new(py, f.into()).into_any().unbind() + } + Type::FLOAT8 => { + let f = postgres_protocol::types::float8_from_sql(raw)?; + PyFloat::new(py, f).into_any().unbind() + } + Type::TEXT | Type::VARCHAR | Type::NAME | Type::BPCHAR => { + PyString::from_bytes(py, raw)?.into_any().unbind() + } + Type::BYTEA => PyBytes::new(py, raw).into_any().unbind(), + _ => { + // This should never happen, unless the `accepts` method is out + // of sync. + return Err(format!("unsupported column type for postgres: {ty}").into()); + } + }; + + Ok(PythonPgFromSql(Some(obj))) + } +} + +#[cfg(test)] +mod tests { + //! These tests exercise the value mapping in isolation — no live Postgres + //! server is needed. The `to_sql` / `from_sql` halves both operate on raw + //! byte buffers, so we can drive them directly with hand-built buffers and + //! a chosen column [`Type`], and the `from_py` classifier just needs a GIL. + + use super::*; + + /// Encode `value` for column type `ty`, returning the wire bytes and the + /// `IsNull` flag. Panics on a `ToSql` error so callers can assert on the + /// happy path concisely. + fn encode(value: &PgValue, ty: &Type) -> (Vec, bool) { + let mut buf = BytesMut::new(); + let is_null = value.to_sql(ty, &mut buf).expect("encoding should succeed"); + (buf.to_vec(), matches!(is_null, IsNull::Yes)) + } + + /// Like [`encode`], but surfaces the `ToSql` result so a test can assert on + /// the error path (e.g. an out-of-range integer). + fn encode_result(value: &PgValue, ty: &Type) -> Result<(), Box> { + let mut buf = BytesMut::new(); + value.to_sql(ty, &mut buf).map(|_| ()) + } + + #[test] + fn from_py_classifies_supported_types() { + Python::initialize(); + Python::attach(|py| { + // `None` -> NULL. + assert!(matches!( + PgValue::from_py(&py.None().into_bound(py)).unwrap(), + PgValue::Null + )); + + // `bool` must win over `int` (it is an `int` subclass in Python). + assert!(matches!( + PgValue::from_py(&true.into_pyobject(py).unwrap()).unwrap(), + PgValue::Bool(true) + )); + + assert!(matches!( + PgValue::from_py(&7i64.into_pyobject(py).unwrap()).unwrap(), + PgValue::Int(7) + )); + + assert!(matches!( + PgValue::from_py(&1.5f64.into_pyobject(py).unwrap()).unwrap(), + PgValue::Float(v) if v == 1.5 + )); + + match PgValue::from_py(&"hello".into_pyobject(py).unwrap()).unwrap() { + PgValue::Text(s) => assert_eq!(&*s, "hello"), + other => panic!("expected Text, got {other:?}"), + } + + match PgValue::from_py(&PyBytes::new(py, b"\x00\xff").into_any()).unwrap() { + PgValue::Bytea(b) => assert_eq!(&*b, b"\x00\xff"), + other => panic!("expected Bytea, got {other:?}"), + } + }); + } + + #[test] + fn from_py_extracts_via_frompyobject() { + // The cursor binds parameters by extracting `PgValue` straight off the + // Python argument; check that `FromPyObject` path forwards to `from_py`. + Python::initialize(); + Python::attach(|py| { + let obj = 7i64.into_pyobject(py).unwrap().into_any(); + assert!(matches!(obj.extract::().unwrap(), PgValue::Int(7))); + }); + } + + #[test] + fn from_py_rejects_unsupported_type() { + Python::initialize(); + Python::attach(|py| { + // A list is not a scalar we know how to bind. + let list = pyo3::types::PyList::new(py, [1, 2, 3]).unwrap(); + let err = PgValue::from_py(&list.into_any()).unwrap_err(); + assert!(err.is_instance_of::(py)); + // The message names the offending type, which is the useful part. + assert!(err.to_string().contains("list"), "got: {err}"); + }); + } + + #[test] + fn to_sql_encodes_each_type_for_its_column() { + // NULL is encoded as "no bytes, IsNull::Yes" regardless of column type. + let (bytes, is_null) = encode(&PgValue::Null, &Type::INT4); + assert!(is_null); + assert!(bytes.is_empty()); + + // Integers are width-specific: the same `Int` encodes to 2/4/8 bytes + // depending on the column type. + assert_eq!(encode(&PgValue::Int(1), &Type::INT2).0, 1i16.to_be_bytes()); + assert_eq!(encode(&PgValue::Int(1), &Type::INT4).0, 1i32.to_be_bytes()); + assert_eq!(encode(&PgValue::Int(1), &Type::INT8).0, 1i64.to_be_bytes()); + + assert_eq!(encode(&PgValue::Bool(true), &Type::BOOL).0, vec![1]); + assert_eq!(encode(&PgValue::Bool(false), &Type::BOOL).0, vec![0]); + assert_eq!( + encode(&PgValue::Float(1.0), &Type::FLOAT8).0, + 1.0f64.to_be_bytes() + ); + assert_eq!( + encode(&PgValue::Text("hi".into()), &Type::TEXT).0, + b"hi".to_vec() + ); + assert_eq!( + encode(&PgValue::Bytea(Box::from(&b"\x01\x02"[..])), &Type::BYTEA).0, + vec![1, 2] + ); + } + + #[test] + fn to_sql_float4_narrows_with_precision_loss() { + // `0.1` is not representable in binary floating point, so the f64 and + // f32 encodings genuinely differ. Encoding to a FLOAT4 column must use + // the (lossy) f32 narrowing, not reinterpret the f64 bytes. + let value = 0.1f64; + assert_eq!( + encode(&PgValue::Float(value), &Type::FLOAT4).0, + (value as f32).to_be_bytes() + ); + // And the result is provably narrower than the FLOAT8 encoding. + assert_ne!( + encode(&PgValue::Float(value), &Type::FLOAT4).0, + value.to_be_bytes()[..4].to_vec() + ); + } + + #[test] + fn to_sql_rejects_mismatched_column_type() { + // A value whose type doesn't match the column is a `WrongType` error + // rather than a silent reinterpretation. Hit a few distinct arms. + let cases: &[(PgValue, Type)] = &[ + (PgValue::Int(1), Type::TEXT), + (PgValue::Text("x".into()), Type::INT4), + (PgValue::Bytea(Box::from(&b"x"[..])), Type::TEXT), + (PgValue::Bool(true), Type::INT4), + (PgValue::Float(1.0), Type::INT8), + ]; + for (value, ty) in cases { + let mut buf = BytesMut::new(); + assert!( + value.to_sql(ty, &mut buf).is_err(), + "expected {value:?} -> {ty} to be rejected" + ); + } + } + + #[test] + fn to_sql_integer_width_boundaries() { + // Boundary values for each width encode; one step past the boundary is + // rejected by the `try_into` guards (INT8 spans all of i64, so there is + // nothing out of range for it). + assert!(encode_result(&PgValue::Int(i16::MAX as i64), &Type::INT2).is_ok()); + assert!(encode_result(&PgValue::Int(i16::MIN as i64), &Type::INT2).is_ok()); + assert!(encode_result(&PgValue::Int(i16::MAX as i64 + 1), &Type::INT2).is_err()); + assert!(encode_result(&PgValue::Int(i16::MIN as i64 - 1), &Type::INT2).is_err()); + + assert!(encode_result(&PgValue::Int(i32::MAX as i64), &Type::INT4).is_ok()); + assert!(encode_result(&PgValue::Int(i32::MIN as i64), &Type::INT4).is_ok()); + assert!(encode_result(&PgValue::Int(i32::MAX as i64 + 1), &Type::INT4).is_err()); + assert!(encode_result(&PgValue::Int(i32::MIN as i64 - 1), &Type::INT4).is_err()); + + assert_eq!( + encode(&PgValue::Int(i64::MAX), &Type::INT8).0, + i64::MAX.to_be_bytes() + ); + assert_eq!( + encode(&PgValue::Int(i64::MIN), &Type::INT8).0, + i64::MIN.to_be_bytes() + ); + } + + #[test] + fn accepts_lists_match_supported_types() { + // The encode and decode sides must accept exactly the supported column + // types and reject everything else; a drift between the two `accepts` + // lists and the `match` arms would surface here. + for ty in [ + Type::BOOL, + Type::INT2, + Type::INT4, + Type::INT8, + Type::FLOAT4, + Type::FLOAT8, + Type::TEXT, + Type::VARCHAR, + Type::NAME, + Type::BPCHAR, + Type::BYTEA, + ] { + assert!(::accepts(&ty), "ToSql should accept {ty}"); + assert!( + ::accepts(&ty), + "FromSql should accept {ty}" + ); + } + + for ty in [Type::JSON, Type::TIMESTAMPTZ, Type::UUID] { + assert!( + !::accepts(&ty), + "ToSql should reject {ty}" + ); + assert!( + !::accepts(&ty), + "FromSql should reject {ty}" + ); + } + } + + #[test] + fn from_sql_decodes_into_python_objects() { + use tokio_postgres::types::FromSql; + + Python::initialize(); + Python::attach(|py| { + // Each supported type round-trips from its wire bytes to the + // matching Python object. This mirrors the encode test's type list. + let int2 = PythonPgFromSql::from_sql(&Type::INT2, &7i16.to_be_bytes()).unwrap(); + assert_eq!(int2.0.unwrap().extract::(py).unwrap(), 7); + + let int4 = PythonPgFromSql::from_sql(&Type::INT4, &42i32.to_be_bytes()).unwrap(); + assert_eq!(int4.0.unwrap().extract::(py).unwrap(), 42); + + // A wide INT8 that wouldn't fit in INT4, to prove the width is honoured. + let big = i64::MAX - 1; + let int8 = PythonPgFromSql::from_sql(&Type::INT8, &big.to_be_bytes()).unwrap(); + assert_eq!(int8.0.unwrap().extract::(py).unwrap(), big); + + // FLOAT4 decodes via an f32, then widens to a Python float. + let f4 = PythonPgFromSql::from_sql(&Type::FLOAT4, &1.5f32.to_be_bytes()).unwrap(); + assert_eq!(f4.0.unwrap().extract::(py).unwrap(), 1.5); + + let f8 = PythonPgFromSql::from_sql(&Type::FLOAT8, &2.5f64.to_be_bytes()).unwrap(); + assert_eq!(f8.0.unwrap().extract::(py).unwrap(), 2.5); + + let b = PythonPgFromSql::from_sql(&Type::BOOL, &[1]).unwrap(); + assert!(b.0.unwrap().extract::(py).unwrap()); + + let text = PythonPgFromSql::from_sql(&Type::TEXT, b"hi").unwrap(); + assert_eq!(text.0.unwrap().extract::(py).unwrap(), "hi"); + + // BYTEA passes raw bytes through unchanged, including NULs and + // non-UTF-8 data — the natural inverse of the encode test. + let bytea = PythonPgFromSql::from_sql(&Type::BYTEA, b"\x00\xff").unwrap(); + assert_eq!( + bytea.0.unwrap().extract::>(py).unwrap(), + b"\x00\xff" + ); + + // A SQL NULL decodes to `None` via `from_sql_null`. + let null = PythonPgFromSql::from_sql_null(&Type::INT4).unwrap(); + assert!(null.0.is_none()); + }); + } + + #[test] + fn from_sql_rejects_unsupported_column_type() { + use tokio_postgres::types::FromSql; + + // The `_ =>` arm in `from_sql_with_py` guards against `accepts` drifting + // out of sync; decoding an unsupported type is an error, not a panic. + Python::initialize(); + Python::attach(|_py| { + assert!(PythonPgFromSql::from_sql(&Type::JSON, b"{}").is_err()); + }); + } + + #[test] + fn from_sql_rejects_non_utf8_text() { + use tokio_postgres::types::FromSql; + + // TEXT decode goes through `PyString::from_bytes`, which validates + // UTF-8: malformed bytes surface as an error rather than a panic. + Python::initialize(); + Python::attach(|_py| { + assert!(PythonPgFromSql::from_sql(&Type::TEXT, b"\xff\xfe").is_err()); + }); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 28783afbba..e7714301f5 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -7,6 +7,7 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; pub mod config; +pub mod database; pub mod deferred; pub mod duration; pub mod errors; @@ -71,6 +72,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { acl::register_module(py, m)?; deferred::register_module(py, m)?; + database::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; handlers::register_module(py, m)?;