Fix parsing events that have large integers (#19819)

Follow on from https://github.com/element-hq/synapse/pull/19701.

Unfortunately serde has a bug when using `#[serde(flatten)]` with
`arbitrary-precision` feature when handling integers that fit in a i128
when doing `serde_json::from_value`. See
https://github.com/serde-rs/serde/issues/2230.

The `depythonize` hits the same issue. To fix this we make it so we only
parse events from strings and not values.

rust/src/events/formats/mod.rs

@@ -98,6 +98,13 @@ pub use vmsc4242::EventFormatVMSC4242;
 /// fields as they are mutable (and must be deep-copied if the event is cloned).
 /// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that
 /// the serialised JSON is a single flat object matching the Matrix spec.
+///
+/// Note, deserialization of this struct must not be done from
+/// [`serde_json::Value`] nor [`pythonize::depythonize`], due to a bug with
+/// `#[serde(flatten)]` combined with the `arbitrary_precision` feature.
+/// Instead, deserialize directly from a JSON string with
+/// `serde_json::from_str`. See https://github.com/serde-rs/serde/issues/2230
+/// for details.
 #[derive(Serialize, Deserialize)]
 pub struct FormattedEvent<E = Arc<EventFormatEnum>> {
     /// The event's signatures.

rust/src/events/mod.rs

@@ -160,10 +160,15 @@ pub struct Event {
 
 #[pymethods]
 impl Event {
+    /// Construct an Event from a JSON string, room version, and internal
+    /// metadata dict.
+    ///
+    /// We do no accept a Python dict directly because of the issues with
+    /// depythonize and large integers (see [`FormattedEvent`] for details).
     #[new]
     fn new_from_py<'a, 'py>(
         py: Python<'py>,
-        event_dict: &'a Bound<'py, PyAny>,
+        event_json: &str,
         room_version: &'a Bound<'py, PyAny>,
         internal_metadata_dict: &'a Bound<'py, PyDict>,
         rejected_reason: Option<String>,
@@ -178,14 +183,14 @@ impl Event {
 
         let rejected_reason = rejected_reason.map(String::into_boxed_str);
 
-        // Parse the event dict into a FormattedEvent, converting any failures to
+        // Parse the event JSON into a FormattedEvent, converting any failures to
         // a `ValueError`.
-        let parsed_event = depythonize_event_dict(room_version, event_dict).map_err(|err| {
+        let parsed_event = event_dict_from_json_str(room_version, event_json).map_err(|err| {
             let new_err = PyValueError::new_err(format!(
-                "Failed to parse event for room version {}",
-                room_version
+                "Failed to parse event for room version {}: {}",
+                room_version, err
             ));
-            new_err.set_cause(py, Some(err));
+            new_err.set_cause(py, Some(PyValueError::new_err(err.to_string())));
             new_err
         })?;
 
@@ -555,63 +560,27 @@ impl Event {
     }
 }
 
-fn depythonize_event_dict(
+/// Parses a JSON string into a [`FormattedEvent`] for the given room version.
+fn event_dict_from_json_str(
     room_version: &RoomVersion,
-    event_dict: &Bound<'_, PyAny>,
-) -> PyResult<FormattedEvent> {
-    let formatted_event: FormattedEvent = match room_version.event_format {
-        EventFormatVersions::ROOM_V1_V2 => {
-            let event_format: FormattedEvent<EventFormatV1> = depythonize(event_dict)?;
-
-            event_format.into()
-        }
-        EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => {
-            let event_format: FormattedEvent<EventFormatV2V3> = depythonize(event_dict)?;
-            event_format.into()
-        }
-        EventFormatVersions::ROOM_V11_HYDRA_PLUS => {
-            let event_format: FormattedEvent<EventFormatV4> = depythonize(event_dict)?;
-            event_format.into()
-        }
-        EventFormatVersions::ROOM_VMSC4242 => {
-            let event_format: FormattedEvent<EventFormatVMSC4242> = depythonize(event_dict)?;
-            event_format.into()
-        }
-        _ => {
-            return Err(PyValueError::new_err(format!(
-                "Unsupported room version: {}",
-                room_version
-            )))
-        }
-    };
-
-    formatted_event.validate()?;
-
-    Ok(formatted_event)
-}
-
-/// Converts an event dict as [`serde_json::Value`] into a [`FormattedEvent`].
-fn event_dict_from_json_value(
-    room_version: &RoomVersion,
-    event_dict: serde_json::Value,
+    event_json: &str,
 ) -> Result<FormattedEvent, Error> {
     let formatted_event: FormattedEvent = match room_version.event_format {
         EventFormatVersions::ROOM_V1_V2 => {
-            let event_format: FormattedEvent<EventFormatV1> = serde_json::from_value(event_dict)?;
-
+            let event_format: FormattedEvent<EventFormatV1> = serde_json::from_str(event_json)?;
             event_format.into()
         }
         EventFormatVersions::ROOM_V3 | EventFormatVersions::ROOM_V4_PLUS => {
-            let event_format: FormattedEvent<EventFormatV2V3> = serde_json::from_value(event_dict)?;
+            let event_format: FormattedEvent<EventFormatV2V3> = serde_json::from_str(event_json)?;
             event_format.into()
         }
         EventFormatVersions::ROOM_V11_HYDRA_PLUS => {
-            let event_format: FormattedEvent<EventFormatV4> = serde_json::from_value(event_dict)?;
+            let event_format: FormattedEvent<EventFormatV4> = serde_json::from_str(event_json)?;
             event_format.into()
         }
         EventFormatVersions::ROOM_VMSC4242 => {
             let event_format: FormattedEvent<EventFormatVMSC4242> =
-                serde_json::from_value(event_dict)?;
+                serde_json::from_str(event_json)?;
             event_format.into()
         }
         _ => {
@@ -638,10 +607,17 @@ fn redact_event_py(event: &Event) -> PyResult<Event> {
     })?;
 
     let redacted_value = redact(&event_value, event.room_version)?;
-    let redacted_formatted_event = event_dict_from_json_value(event.room_version, redacted_value)
-        .map_err(|err| {
-        PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err))
+
+    // We can't convert from a value into [`Event`] directly, so we round-trip
+    // through JSON. See [`FormattedEvent`] for details on why we can't go
+    // directly through Python dicts.
+    let redacted_event_json = serde_json::to_string(&redacted_value).map_err(|err| {
+        PyValueError::new_err(format!("Failed to serialize redacted event: {}", err))
     })?;
+    let redacted_formatted_event =
+        event_dict_from_json_str(event.room_version, &redacted_event_json).map_err(|err| {
+            PyValueError::new_err(format!("Failed to deserialize redacted event: {}", err))
+        })?;
 
     let redacted_event = Event {
         parsed_event: redacted_formatted_event,

rust/src/events/unsigned.rs

@@ -16,9 +16,9 @@
 use std::sync::{Arc, RwLock, RwLockReadGuard};
 
 use pyo3::{
-    exceptions::{PyKeyError, PyRuntimeError, PyTypeError},
+    exceptions::{PyKeyError, PyRuntimeError, PyTypeError, PyValueError},
     pyclass, pymethods,
-    types::{PyAnyMethods, PyList, PyListMethods, PyMapping},
+    types::{PyAnyMethods, PyList, PyListMethods},
     Bound, IntoPyObjectExt, PyAny, PyResult, Python,
 };
 use pythonize::{depythonize, pythonize};
@@ -114,9 +114,14 @@ impl Unsigned {
 
 #[pymethods]
 impl Unsigned {
+    /// Create a new `Unsigned` from a JSON string.
+    ///
+    /// We do no accept a Python dict directly because of the issues with
+    /// depythonize and large integers (see [`FormattedEvent`] for details).
     #[new]
-    fn py_new(unsigned: Bound<'_, PyMapping>) -> PyResult<Self> {
-        let inner = depythonize(&unsigned)?;
+    fn py_new(unsigned_json: &str) -> PyResult<Self> {
+        let inner = serde_json::from_str(unsigned_json)
+            .map_err(|err| PyValueError::new_err(format!("Failed to parse unsigned: {}", err)))?;
 
         Ok(Self {
             inner: Arc::new(RwLock::new(inner)),