docs: Reference latest version of matrix spec

changelog.d/2107.doc.md

@@ -0,0 +1 @@
+Added missing documentation to config parameters. Contributed by @renegadespork

conduwuit-example.toml

@@ -170,43 +170,136 @@
 #
 #db_write_buffer_capacity_mb = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# The maximum number of Persisted Data Units (PDUs) to cache.
+# Accepts any 32-bit integer.
+#
+# PDUs are events broadcast from one homeserver to any others that have
+# joined the same room (identified by Room ID). They are persisted in
+# long-term storage and record the history of messages and state
+# for a room.
+#
+# An example of a PDU would be a message or joining a room.
+#
+# Setting this higher might be useful if the server has a large memory
+# capacity and is noticeably I/O limited.
+#
+# This defaults to 100,000 + (10,000 * CPU core count)
 #
 #pdu_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# The maximum number of auth chains to cache.
+# Accepts any 32-bit integer.
+#
+# When determining is a user is authorized to send an event,
+# the server will first check the cache for an auth chain.
+# If the auth chain is not present in the cache,
+# The server will read the auth chain from the database
+# and add it to the cache.
+#
+# This defaults to 100,000 + (10,000 * CPU core count)
 #
 #auth_chain_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for eventid data references.
+# Accepts any 32-bit integer.
+#
+# Each matrix event can be referenced in the database via either
+# an eventid (string) or a shorteventid (64-bit integer).
+# This caps the capacity of cached eventids referenced
+# by their shorteventid.
+#
+# This defaults to 100,000 + (50,000 * CPU core count)
 #
 #shorteventid_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for eventid data references.
+# Accepts any 32-bit integer.
+#
+# Each matrix event can be referenced in the database via either
+# an eventid (string) or a shorteventid (64-bit integer).
+# This caps the capacity of cached shorteventids referenced
+# by their eventid.
+#
+# This defaults to 100,000 + (25,000 * CPU core count)
 #
 #eventidshort_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for pdu data references.
+# Accepts any 32-bit integer.
+#
+# This caps the capacity of cached pdus referenced
+# by their eventid.
+#
+# This defaults to 100,000 + (25,000 * CPU core count)
 #
 #eventid_pdu_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for state key data references.
+# Accepts any 32-bit integer.
+#
+# State keys are used by the server to verify that a token
+# from a third-party invite event was actually generated
+# by the server where the room resides.
+#
+# These state keys can be referenced in the database via either
+# a statekey (string) or a shortstatekey (64-bit integer).
+# This caps the capacity of cached statekeys referenced
+# by shortstatekeys.
+#
+# This defaults to 100,000 + (10,000 * CPU core count)
 #
 #shortstatekey_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for state key data references.
+# Accepts any 32-bit integer.
+#
+# State keys are used by the server to verify that a token
+# from a third-party invite event was actually generated
+# by the server where the room resides.
+#
+# These state keys can be referenced in the database via either
+# a statekey (string) or a shortstatekey (64-bit integer).
+# This caps the capacity of cached shortstatekeys referenced
+# by statekeys.
+#
+# This defaults to 100,000 + (10,000 * CPU core count)
 #
 #statekeyshort_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for event_data references scoped
+# by server_name.
+# Accepts any 32-bit integer.
+#
+# Event data can be referenced in the database by server name.
+# This is useful if the server needs to clear all events associated
+# with a particular federated server, for example.
+# This caps the capacity of cached event_data referenced
+# by server_name.
+#
+# This defaults to 500,000 + (100,000 * CPU core count)
 #
 #servernameevent_data_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for room states.
+# Accepts any 32-bit integer.
+#
+# To ensure data integrity of room events across asynchronous
+# requests from federated servers, Matrix tracks the state
+# of a room using a statehash and the changes since the
+# previous (parent) statehash.
+# This caps the capacity of cached room state data.
+#
+# This defaults to 100 * CPU core count.
 #
 #stateinfo_cache_capacity = varies by system
 
-# This item is undocumented. Please contribute documentation for it.
+# Determines the cache size for spacehierarchy data.
+# Accepts any 32-bit integer.
+#
+# Each roomid contains info on its place in the space hierarchy.
+# This caps the capacity of cached spacehierarchy data.
+#
+# This defaults to 1,000 * CPU core count.
 #
 #roomid_spacehierarchy_cache_capacity = varies by system
 
@@ -347,7 +440,13 @@
 #
 #max_request_size = 20971520
 
-# This item is undocumented. Please contribute documentation for it.
+# Maximum number of prev_events the server will request from
+# other servers.
+#
+# When requesting room events from another server, this server will
+# specify a maximum prev_events for the other server to return in its
+# response. This can be used for things like backfilling room data and
+# getting missing events.
 #
 #max_fetch_prev_events = 1024
 
@@ -455,28 +554,9 @@
 #
 #sender_idle_timeout = 180
 
-# Federation sender retry backoff base (seconds).
+# Federation sender transaction retry backoff limit (seconds).
 #
-# This period will be doubled for each failed federation request until
-# either the remote server becomes healthy, or the value is clamped to
-# `sender_retry_backoff_limit`.
-#
-#sender_retry_backoff_base = 60
-
-# Federation sender retry backoff limit (seconds).
-#
-# Defaults to one week. Requests will never stop being retried if their
-# backoff period exceeds this value, however the maximum amount of time
-# between each request will instead be clamped at this value.
-#
-# The backoff period is reset if a successful request is made, or
-# continuwuity receives a request from the server that is being backed off
-# from.
-#
-# It is not recommended to lower this value below 48 hours or above
-# 1 year.
-#
-#sender_retry_backoff_limit = 806400
+#sender_retry_backoff_limit = 86400
 
 # Appservice URL request connection timeout. Defaults to 35 seconds as
 # generally appservices are hosted within the same network.
@@ -751,11 +831,15 @@
 #
 #tracing_flame = false
 
-# This item is undocumented. Please contribute documentation for it.
+# Which log level tracing_flame will trace.
+#
+# See "tracing_flame" for more details.
 #
 #tracing_flame_filter = "info"
 
-# This item is undocumented. Please contribute documentation for it.
+# The output path for tracing_flame data.
+#
+# See "tracing_flame" for more details.
 #
 #tracing_flame_output_path = "./tracing.folded"
 
@@ -983,7 +1067,7 @@
 #
 #rocksdb_log_level = "error"
 
-# This item is undocumented. Please contribute documentation for it.
+# Whether to output RocksDB errors to stderr.
 #
 #rocksdb_log_stderr = false
 
@@ -1233,7 +1317,17 @@
 #
 #emergency_password =
 
-# This item is undocumented. Please contribute documentation for it.
+# Specifies the path where push notifications will be sent to a
+# push gateway.
+#
+# When a Matrix client registers for push notifications, it will specify
+# a push gateway url where notification events will be sent.
+# According to the current spec for Matrix Push Gateways, the path should
+# always be "/_matrix/push/v1/notify".
+# You should only change this if you have configured your own push gateway
+# that requires a different path.
+#
+# For more information on Push Gateways: https://spec.matrix.org/latest/push-gateway-api/
 #
 #notification_push_path = "/_matrix/push/v1/notify"
 
@@ -1390,7 +1484,10 @@
 #
 #allow_legacy_media = true
 
-# This item is undocumented. Please contribute documentation for it.
+# If set to true, prevents fetching new legacy remote media.
+#
+# Legacy media may still be accessible if it was already fetched
+# previously and allow_legacy_media is true.
 #
 #freeze_legacy_media = true
 
@@ -1798,7 +1895,18 @@
 #
 #tokio_console = false
 
-# This item is undocumented. Please contribute documentation for it.
+# A list of flags that modify startup behavior.
+# Accepts an array of strings.
+#
+# During startup, the program will check for the presence of certain
+# strings in this array. Matching values will modify how the program runs
+# those startup tasks.
+#
+# If this variable is empty or contains no matches, the server will start
+# normally.
+#
+# Currently this only checks for "smoke" which enables "Smoketest mode."
+# This mode exits the program after running startup tasks.
 #
 #test = false
 

src/api/router/auth.rs

@@ -149,9 +149,6 @@ async fn verify<B: AsRef<[u8]> + Sync>(
 					)));
 				}
 
-				// Ping the server as healthy
-				services.federation.mark_healthy(&output.origin);
-
 				Ok(output.origin)
 			},
 			| Err(err) =>

src/core/config/mod.rs

@@ -238,42 +238,155 @@ pub struct Config {
 	#[serde(default = "default_db_write_buffer_capacity_mb")]
 	pub db_write_buffer_capacity_mb: f64,
 
+	/// The maximum number of Persisted Data Units (PDUs) to cache.
+	/// Accepts any 32-bit integer.
+	///
+	/// PDUs are events broadcast from one homeserver to any others that have
+	/// joined the same room (identified by Room ID). They are persisted in
+	/// long-term storage and record the history of messages and state
+	/// for a room.
+	///
+	/// An example of a PDU would be a message or joining a room.
+	///
+	/// Setting this higher might be useful if the server has a large memory
+	/// capacity and is noticeably I/O limited.
+	///
+	/// This defaults to 100,000 + (10,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_pdu_cache_capacity")]
 	pub pdu_cache_capacity: u32,
 
+	/// The maximum number of auth chains to cache.
+	/// Accepts any 32-bit integer.
+	///
+	/// When determining is a user is authorized to send an event,
+	/// the server will first check the cache for an auth chain.
+	/// If the auth chain is not present in the cache,
+	/// The server will read the auth chain from the database
+	/// and add it to the cache.
+	///
+	/// This defaults to 100,000 + (10,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_auth_chain_cache_capacity")]
 	pub auth_chain_cache_capacity: u32,
 
+	/// Determines the cache size for eventid data references.
+	/// Accepts any 32-bit integer.
+	///
+	/// Each matrix event can be referenced in the database via either
+	/// an eventid (string) or a shorteventid (64-bit integer).
+	/// This caps the capacity of cached eventids referenced
+	/// by their shorteventid.
+	///
+	/// This defaults to 100,000 + (50,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_shorteventid_cache_capacity")]
 	pub shorteventid_cache_capacity: u32,
 
+	/// Determines the cache size for eventid data references.
+	/// Accepts any 32-bit integer.
+	///
+	/// Each matrix event can be referenced in the database via either
+	/// an eventid (string) or a shorteventid (64-bit integer).
+	/// This caps the capacity of cached shorteventids referenced
+	/// by their eventid.
+	///
+	/// This defaults to 100,000 + (25,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_eventidshort_cache_capacity")]
 	pub eventidshort_cache_capacity: u32,
 
+	/// Determines the cache size for pdu data references.
+	/// Accepts any 32-bit integer.
+	///
+	/// This caps the capacity of cached pdus referenced
+	/// by their eventid.
+	///
+	/// This defaults to 100,000 + (25,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_eventid_pdu_cache_capacity")]
 	pub eventid_pdu_cache_capacity: u32,
 
+	/// Determines the cache size for state key data references.
+	/// Accepts any 32-bit integer.
+	///
+	/// State keys are used by the server to verify that a token
+	/// from a third-party invite event was actually generated
+	/// by the server where the room resides.
+	///
+	/// These state keys can be referenced in the database via either
+	/// a statekey (string) or a shortstatekey (64-bit integer).
+	/// This caps the capacity of cached statekeys referenced
+	/// by shortstatekeys.
+	///
+	/// This defaults to 100,000 + (10,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_shortstatekey_cache_capacity")]
 	pub shortstatekey_cache_capacity: u32,
 
+	/// Determines the cache size for state key data references.
+	/// Accepts any 32-bit integer.
+	///
+	/// State keys are used by the server to verify that a token
+	/// from a third-party invite event was actually generated
+	/// by the server where the room resides.
+	///
+	/// These state keys can be referenced in the database via either
+	/// a statekey (string) or a shortstatekey (64-bit integer).
+	/// This caps the capacity of cached shortstatekeys referenced
+	/// by statekeys.
+	///
+	/// This defaults to 100,000 + (10,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_statekeyshort_cache_capacity")]
 	pub statekeyshort_cache_capacity: u32,
 
+	/// Determines the cache size for event_data references scoped
+	/// by server_name.
+	/// Accepts any 32-bit integer.
+	///
+	/// Event data can be referenced in the database by server name.
+	/// This is useful if the server needs to clear all events associated
+	/// with a particular federated server, for example.
+	/// This caps the capacity of cached event_data referenced
+	/// by server_name.
+	///
+	/// This defaults to 500,000 + (100,000 * CPU core count)
+	///
 	/// default: varies by system
 	#[serde(default = "default_servernameevent_data_cache_capacity")]
 	pub servernameevent_data_cache_capacity: u32,
 
+	/// Determines the cache size for room states.
+	/// Accepts any 32-bit integer.
+	///
+	/// To ensure data integrity of room events across asynchronous
+	/// requests from federated servers, Matrix tracks the state
+	/// of a room using a statehash and the changes since the
+	/// previous (parent) statehash.
+	/// This caps the capacity of cached room state data.
+	///
+	/// This defaults to 100 * CPU core count.
+	///
 	/// default: varies by system
 	#[serde(default = "default_stateinfo_cache_capacity")]
 	pub stateinfo_cache_capacity: u32,
 
+	/// Determines the cache size for spacehierarchy data.
+	/// Accepts any 32-bit integer.
+	///
+	/// Each roomid contains info on its place in the space hierarchy.
+	/// This caps the capacity of cached spacehierarchy data.
+	///
+	/// This defaults to 1,000 * CPU core count.
+	///
 	/// default: varies by system
 	#[serde(default = "default_roomid_spacehierarchy_cache_capacity")]
 	pub roomid_spacehierarchy_cache_capacity: u32,
@@ -431,6 +544,14 @@ pub struct Config {
 	#[serde(default = "default_max_request_size")]
 	pub max_request_size: usize,
 
+	/// Maximum number of prev_events the server will request from
+	/// other servers.
+	///
+	/// When requesting room events from another server, this server will
+	/// specify a maximum prev_events for the other server to return in its
+	/// response. This can be used for things like backfilling room data and
+	/// getting missing events.
+	///
 	/// default: 1024
 	#[serde(default = "default_max_fetch_prev_events")]
 	pub max_fetch_prev_events: u16,
@@ -573,30 +694,9 @@ pub struct Config {
 	#[serde(default = "default_sender_idle_timeout")]
 	pub sender_idle_timeout: u64,
 
-	/// Federation sender retry backoff base (seconds).
+	/// Federation sender transaction retry backoff limit (seconds).
 	///
-	/// This period will be doubled for each failed federation request until
-	/// either the remote server becomes healthy, or the value is clamped to
-	/// `sender_retry_backoff_limit`.
-	///
-	/// default: 60
-	#[serde(default = "default_sender_retry_backoff_base")]
-	pub sender_retry_backoff_base: u64,
-
-	/// Federation sender retry backoff limit (seconds).
-	///
-	/// Defaults to one week. Requests will never stop being retried if their
-	/// backoff period exceeds this value, however the maximum amount of time
-	/// between each request will instead be clamped at this value.
-	///
-	/// The backoff period is reset if a successful request is made, or
-	/// continuwuity receives a request from the server that is being backed off
-	/// from.
-	///
-	/// It is not recommended to lower this value below 48 hours or above
-	/// 1 year.
-	///
-	/// default: 806400
+	/// default: 86400
 	#[serde(default = "default_sender_retry_backoff_limit")]
 	pub sender_retry_backoff_limit: u64,
 
@@ -911,13 +1011,23 @@ pub struct Config {
 	///
 	/// [1]: https://github.com/jonhoo/inferno
 	/// [2]: www.speedscope.app
+	///
+	/// default: false
 	#[serde(default)]
 	pub tracing_flame: bool,
 
+	/// Which log level tracing_flame will trace.
+	///
+	/// See "tracing_flame" for more details.
+	///
 	/// default: "info"
 	#[serde(default = "default_tracing_flame_filter")]
 	pub tracing_flame_filter: String,
 
+	/// The output path for tracing_flame data.
+	///
+	/// See "tracing_flame" for more details.
+	///
 	/// default: "./tracing.folded"
 	#[serde(default = "default_tracing_flame_output_path")]
 	pub tracing_flame_output_path: String,
@@ -1179,6 +1289,9 @@ pub struct Config {
 	#[serde(default = "default_rocksdb_log_level")]
 	pub rocksdb_log_level: String,
 
+	/// Whether to output RocksDB errors to stderr.
+	///
+	/// default: false
 	#[serde(default)]
 	pub rocksdb_log_stderr: bool,
 
@@ -1451,6 +1564,18 @@ pub struct Config {
 	/// display: sensitive
 	pub emergency_password: Option<String>,
 
+	/// Specifies the path where push notifications will be sent to a
+	/// push gateway.
+	///
+	/// When a Matrix client registers for push notifications, it will specify
+	/// a push gateway url where notification events will be sent.
+	/// According to the current spec for Matrix Push Gateways, the path should
+	/// always be "/_matrix/push/v1/notify".
+	/// You should only change this if you have configured your own push gateway
+	/// that requires a different path.
+	///
+	/// For more information on Push Gateways: https://spec.matrix.org/latest/push-gateway-api/
+	///
 	/// default: "/_matrix/push/v1/notify"
 	#[serde(default = "default_notification_push_path")]
 	pub notification_push_path: String,
@@ -1617,9 +1742,17 @@ pub struct Config {
 	///
 	/// Defaults to true for now, but this is highly subject to change, likely
 	/// in the next release.
+	///
+	/// default: true
 	#[serde(default = "true_fn")]
 	pub allow_legacy_media: bool,
 
+	/// If set to true, prevents fetching new legacy remote media.
+	///
+	/// Legacy media may still be accessible if it was already fetched
+	/// previously and allow_legacy_media is true.
+	///
+	/// default: true
 	#[serde(default = "true_fn")]
 	pub freeze_legacy_media: bool,
 
@@ -2079,6 +2212,20 @@ pub struct Config {
 	#[serde(default)]
 	pub tokio_console: bool,
 
+	/// A list of flags that modify startup behavior.
+	/// Accepts an array of strings.
+	///
+	/// During startup, the program will check for the presence of certain
+	/// strings in this array. Matching values will modify how the program runs
+	/// those startup tasks.
+	///
+	/// If this variable is empty or contains no matches, the server will start
+	/// normally.
+	///
+	/// Currently this only checks for "smoke" which enables "Smoketest mode."
+	/// This mode exits the program after running startup tasks.
+	///
+	/// default: false
 	#[serde(default)]
 	pub test: BTreeSet<String>,
 
@@ -2833,8 +2980,6 @@ fn default_sender_timeout() -> u64 { 180 }
 
 fn default_sender_idle_timeout() -> u64 { 180 }
 
-fn default_sender_retry_backoff_base() -> u64 { 60 }
-
 fn default_sender_retry_backoff_limit() -> u64 { 86400 }
 
 fn default_appservice_timeout() -> u64 { 35 }

src/core/utils/mod.rs

@@ -38,8 +38,8 @@
 	string::{str_from_bytes, string_from_bytes},
 	sys::compute::available_parallelism,
 	time::{
-		exponential_backoff::should_continue_backoff, now_millis as millis_since_unix_epoch,
-		timepoint_ago, timepoint_from_now,
+		exponential_backoff::{continue_exponential_backoff, continue_exponential_backoff_secs},
+		now_millis as millis_since_unix_epoch, timepoint_ago, timepoint_from_now,
 	},
 };
 

src/core/utils/time/exponential_backoff.rs

@@ -1,23 +1,38 @@
-use std::time::Duration;
+use std::{cmp, time::Duration};
 
-/// Returns false if the backoff interval has expired based on the inputs,
-/// meaning the operation should be retried.
+/// Returns false if the exponential backoff has expired based on the inputs
 #[inline]
 #[must_use]
-pub fn should_continue_backoff(
+pub fn continue_exponential_backoff_secs(
+	min: u64,
+	max: u64,
+	elapsed: Duration,
+	tries: u32,
+) -> bool {
+	let min = Duration::from_secs(min);
+	let max = Duration::from_secs(max);
+	continue_exponential_backoff(min, max, elapsed, tries)
+}
+
+/// Returns false if the exponential backoff has expired based on the inputs
+#[inline]
+#[must_use]
+pub fn continue_exponential_backoff(
 	min: Duration,
 	max: Duration,
 	elapsed: Duration,
 	tries: u32,
 ) -> bool {
-	elapsed < next_interval(min, max, tries)
+	let min = min.saturating_mul(tries).saturating_mul(tries);
+	let min = cmp::min(min, max);
+	elapsed < min
 }
 
-/// Determines the interval that should be waited before retrying the operation
-/// using the algorithm: `(min * retries).min(max)`.
+/// Determines the minimum number of backoff seconds
 #[must_use]
-#[inline]
-pub fn next_interval(min: Duration, max: Duration, retries: u32) -> Duration {
-	// TODO(nex): jitter?
-	min.saturating_mul(retries).min(max)
+pub fn min_exp_backoff_duration(min: u64, max: u64, retries: u32) -> Duration {
+	let min = Duration::from_secs(min)
+		.saturating_mul(retries)
+		.saturating_mul(retries);
+	Duration::from_secs(max).min(min)
 }

src/service/federation/execute.rs

@@ -131,8 +131,6 @@ pub async fn execute_on<'i, T, PathBuilderInput>(
 			))));
 		}
 
-		self.ensure_remote_is_healthy(dest)?;
-
 		let actual = self
 			.services
 			.client
@@ -185,13 +183,8 @@ async fn perform<T>(
 			| Ok(response) =>
 				self.handle_response::<T>(dest, actual, &method, &url, response)
 					.await,
-			| Err(error) => {
-				if error.is_connect() {
-					debug_info!("{dest} is unhealthy due to a connect error");
-					self.hit_unhealthy(dest.to_owned());
-				}
-				Err(handle_error(actual, &method, &url, error).expect_err("always returns error"))
-			},
+			| Err(error) =>
+				Err(handle_error(actual, &method, &url, error).expect_err("always returns error")),
 		}
 	}
 
@@ -244,7 +237,6 @@ async fn handle_response<T>(
 			parts,
 			body.as_ref(),
 		))
-		.inspect(|_| self.mark_healthy(dest))
 		.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}")))
 	}
 }

src/service/federation/mod.rs

@@ -1,25 +1,13 @@
 mod execute;
+use std::sync::Arc;
 
-use std::{collections::HashMap, sync::Arc, time::Duration};
-
-use assign::assign;
-use async_trait::async_trait;
-use conduwuit::{
-	Error, Result, Server, SyncRwLock, debug,
-	utils::{math::Expected, millis_since_unix_epoch, time::exponential_backoff::next_interval},
-};
+use conduwuit::{Result, Server};
 pub(crate) use execute::FederationPathBuilderInput;
-use http::StatusCode;
-use ruma::{
-	OwnedServerName, ServerName,
-	api::error::{ErrorKind, LimitExceededErrorData, RetryAfter},
-};
 
 use crate::{Dep, client, moderation, server_keys};
 
 pub struct Service {
 	services: Services,
-	remote_health: SyncRwLock<HashMap<OwnedServerName, (u32, u64)>>,
 }
 
 struct Services {
@@ -29,7 +17,6 @@ struct Services {
 	moderation: Dep<moderation::Service>,
 }
 
-#[async_trait]
 impl crate::Service for Service {
 	fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
 		Ok(Arc::new(Self {
@@ -39,103 +26,8 @@ fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
 				server_keys: args.depend::<server_keys::Service>("server_keys"),
 				moderation: args.depend::<moderation::Service>("moderation"),
 			},
-			remote_health: SyncRwLock::new(HashMap::new()),
 		}))
 	}
 
 	fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
-
-	async fn clear_cache(&self) {
-		let mut map = self.remote_health.write();
-		map.clear();
-	}
-}
-
-impl Service {
-	/// Checks if a remote is "healthy". "Healthy" is defined by either:
-	///
-	/// * The remote has not been marked as having a failed request, OR
-	/// * The next retry timestamp is in the past
-	pub fn is_healthy(&self, server_name: &ServerName) -> bool {
-		let map = self.remote_health.read();
-		let unix_now = millis_since_unix_epoch();
-		if let Some((_, next_retry)) = map.get(server_name) {
-			unix_now >= *next_retry
-		} else {
-			true
-		}
-	}
-
-	/// Returns how long the server should wait before attempting to contact the
-	/// remote again.
-	pub fn retry_after(&self, server_name: &ServerName) -> Option<Duration> {
-		let map = self.remote_health.read();
-		let unix_now = millis_since_unix_epoch();
-		map.get(server_name)
-			.map(|(_, next_retry)| Duration::from_millis((*next_retry).expected_sub(unix_now)))
-	}
-
-	/// Marks or updates a remote's health status as unhealthy. If the remote is
-	/// not already marked as unhealthy, a new entry is created. Otherwise, the
-	/// retry count is incremented and
-	pub fn hit_unhealthy(&self, server_name: OwnedServerName) {
-		let unix_now = millis_since_unix_epoch();
-		let mut map = self.remote_health.write();
-		let sn2 = server_name.clone(); // for logging since map.entry() moves
-		let (retries, next_retry) = map.entry(server_name).or_default();
-		if *next_retry > unix_now {
-			// Don't update the retry marker if we are already in a backoff
-			// period. This prevents the backoff skyrocketing if multiple
-			// concurrent or closely-related requests fail and consequently try
-			// to mark as offline.
-			return;
-		}
-
-		let min = Duration::from_secs(self.services.server.config.sender_retry_backoff_base);
-		let max = Duration::from_secs(self.services.server.config.sender_retry_backoff_limit);
-
-		*retries = retries.saturating_add(1);
-		*next_retry = unix_now.saturating_add(
-			u64::try_from(next_interval(min, max, *retries).as_millis())
-				.expect("backoff milliseconds should not exceed u64::MAX"),
-		);
-		debug!(
-			"{} is (now) unhealthy ({} retries, blocked until: {})",
-			sn2, *retries, *next_retry
-		);
-	}
-
-	/// Marks a server as "healthy" by removing it from the health map.
-	///
-	/// TODO: flush senders too
-	pub fn mark_healthy(&self, server_name: &ServerName) {
-		// TODO: We need to make sure the sender flush DOESN'T trigger if this is called
-		// by the senders themselves.
-		let mut map = self.remote_health.write();
-		if map.remove(server_name).is_some() {
-			debug!("{} is now healthy", server_name);
-		}
-	}
-
-	/// Returns a rate-limited error if the remote is unhealthy.
-	fn ensure_remote_is_healthy(&self, server_name: &ServerName) -> Result<()> {
-		if self.is_healthy(server_name) {
-			Ok(())
-		} else {
-			let retry_after = self
-				.retry_after(server_name)
-				.expect("remote is unhealthy and must have an accompanying retry timestamp");
-			Err(Error::Request(
-				ErrorKind::LimitExceeded(assign!(LimitExceededErrorData::new(), {
-					retry_after: Some(RetryAfter::Delay(retry_after)),
-				})),
-				format!(
-					"Remote server is currently unhealthy (not retrying for another {} seconds)",
-					retry_after.as_secs()
-				)
-				.into(),
-				StatusCode::TOO_MANY_REQUESTS,
-			))
-		}
-	}
 }

src/service/sending/sender.rs

@@ -10,15 +10,14 @@
 
 use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
 use conduwuit::{
-	debug_info, debug_warn, info,
-	utils::{should_continue_backoff, time::exponential_backoff::next_interval},
+	debug_info, debug_warn, info, utils::time::exponential_backoff::min_exp_backoff_duration,
 };
 use conduwuit_core::{
 	Error, Event, Result, at, debug, err, error,
 	matrix::pdu::sticky,
 	result::LogErr,
 	utils::{
-		ReadyExt, calculate_hash,
+		ReadyExt, calculate_hash, continue_exponential_backoff_secs,
 		future::TryExtExt,
 		stream::{BroadbandExt, IterStream, WidebandExt},
 	},
@@ -143,15 +142,7 @@ async fn handle_response<'a>(
 	) {
 		match response {
 			| Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await,
-			| Err((dest, e)) => {
-				if e.status_code().is_server_error()
-					&& let Destination::Federation(dest) = &dest
-				{
-					debug!("{dest} is now unhealthy due to server error response: {e:?}");
-					self.services.federation.hit_unhealthy(dest.clone());
-				}
-				Self::handle_response_err(dest, statuses, &e);
-			},
+			| Err((dest, e)) => Self::handle_response_err(dest, statuses, &e),
 		}
 	}
 
@@ -180,9 +171,6 @@ async fn handle_response_ok<'a>(
 	) {
 		let _cork = self.db.db.cork();
 		self.db.delete_all_active_requests_for(dest).await;
-		if let Destination::Federation(server_name) = dest {
-			self.services.federation.mark_healthy(server_name);
-		}
 
 		// Find events that have been added since starting the last request
 		let new_events = self
@@ -394,28 +382,18 @@ fn should_attempt_send(
 		dest: &Destination,
 		statuses: &mut CurTransactionStatus,
 	) -> Result<(bool, bool)> {
-		if let Destination::Federation(server_name) = dest {
-			let status = statuses
-				.entry(dest.clone())
-				.or_insert(TransactionStatus::Running);
-			let retry = matches!(status, TransactionStatus::Failed(_, _));
-			return match self.services.federation.retry_after(server_name) {
-				| None => Ok((true, retry)),
-				| Some(t) => Ok((t.as_millis() > 0, retry)),
-			};
-		}
-
 		let (mut allow, mut retry) = (true, false);
 		statuses
 			.entry(dest.clone())
 			.and_modify(|e| match e {
 				| TransactionStatus::Failed(tries, time) => {
-					let min = Duration::from_secs(self.server.config.sender_retry_backoff_base);
-					let max = Duration::from_secs(self.server.config.sender_retry_backoff_limit);
-					if should_continue_backoff(min, max, time.elapsed(), *tries)
+					// Fail if a request has failed recently (exponential backoff)
+					let min = self.server.config.sender_timeout;
+					let max = self.server.config.sender_retry_backoff_limit;
+					if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries)
 						&& !matches!(dest, Destination::Appservice(_))
 					{
-						let retry_after = next_interval(min, max, *tries);
+						let retry_after = min_exp_backoff_duration(min, max, *tries);
 						debug_warn!("Not retrying destination for another {retry_after:?}");
 						allow = false;
 					} else {
@@ -468,8 +446,6 @@ async fn select_edus(&self, server_name: &ServerName) -> Result<(EduVec, u64)> {
 		events.extend(presence.into_iter().flatten());
 		events.extend(receipts.into_iter().flatten());
 
-		// TODO(nex): some EDUs like typing need flattening
-
 		Ok((events, max_edu_count.load(Ordering::Acquire)))
 	}