feat: Trace why remotes get health marked

src/service/federation/execute.rs

@@ -187,6 +187,7 @@ async fn perform<T>(
 					.await,
 			| Err(error) => {
 				if error.is_connect() {
+					debug_info!("{dest} is unhealthy due to a connect error");
 					self.hit_unhealthy(dest.to_owned());
 				}
 				Err(handle_error(actual, &method, &url, error).expect_err("always returns error"))

src/service/federation/mod.rs

@@ -4,7 +4,7 @@
 
 use assign::assign;
 use conduwuit::{
-	Error, Result, Server, SyncRwLock,
+	Error, Result, Server, SyncRwLock, debug,
 	utils::{millis_since_unix_epoch, time::exponential_backoff::min_exp_backoff_duration},
 };
 pub(crate) use execute::FederationPathBuilderInput;
@@ -90,6 +90,7 @@ pub fn hit_unhealthy(&self, server_name: OwnedServerName) {
 		// code itself.
 		let unix_now = millis_since_unix_epoch();
 		let mut map = self.remote_health.write();
+		let sn2 = server_name.clone(); // for logging since map.entry() moves
 		let (retries, next_retry) = map.entry(server_name).or_default();
 
 		let min = self.services.server.config.sender_timeout;
@@ -100,6 +101,10 @@ pub fn hit_unhealthy(&self, server_name: OwnedServerName) {
 			u64::try_from(min_exp_backoff_duration(min, max, *retries).as_millis())
 				.expect("backoff milliseconds should not exceed u64::MAX"),
 		);
+		debug!(
+			"{} is (now) unhealthy ({} retries, blocked until: {})",
+			sn2, *retries, *next_retry
+		);
 	}
 
 	/// Marks a server as "healthy" by removing it from the health map.
@@ -109,7 +114,9 @@ pub fn mark_healthy(&self, server_name: &ServerName) {
 		// TODO: We need to make sure the sender flush DOESN'T trigger if this is called
 		// by the senders themselves.
 		let mut map = self.remote_health.write();
-		map.remove(server_name);
+		if map.remove(server_name).is_some() {
+			debug!("{} is now healthy", server_name);
+		}
 	}
 
 	/// Returns a rate-limited error if the remote is unhealthy.

src/service/sending/sender.rs

@@ -146,6 +146,7 @@ async fn handle_response<'a>(
 				if e.status_code().is_server_error()
 					&& let Destination::Federation(dest) = &dest
 				{
+					debug!("{dest} is now unhealthy due to server error response: {e:?}");
 					self.services.federation.hit_unhealthy(dest.clone());
 				}
 				Self::handle_response_err(dest, statuses, &e);