CAD offset safety net

zephcore/adapters/radio/LoRaRadioBase.cpp

@@ -1683,6 +1683,22 @@ int8_t LoRaRadioBase::pickCadProbeLevel()
 	_cad_probe_rr++;
 
 	if (!_cad_auto) {
+		/* The sweep window is ABSOLUTE, so an operating offset parked
+		 * outside it is never probed at all — measured: 49 minutes at
+		 * offset -8 produced 0 probes at -8, which left cadSafetyStep()
+		 * with no evidence and made it a no-op in precisely the
+		 * configuration it exists for (auto off + an offset that cannot
+		 * clear).
+		 *
+		 * When the operator has parked outside the window, probe where
+		 * they actually are instead: the swept curve does not contain
+		 * their operating point, so it cannot inform the hand-tuning it
+		 * was built for either.  Inside the window the sweep already
+		 * covers the operating level and is left exactly as it was. */
+		if (_cad_offset < CAD_SWEEP_MIN || _cad_offset > CAD_SWEEP_MAX) {
+			return _cad_offset;
+		}
+
 		/* Dry-run: even sweep across the observation window so the
 		 * user sees the whole FP-vs-detPeak curve in `get cad`. */
 		int span = CAD_SWEEP_MAX - CAD_SWEEP_MIN + 1;
@@ -1739,24 +1755,16 @@ void LoRaRadioBase::cadStaircaseStep()
 	if (r_op < 0) {
 		return;  /* operating level not warm yet — no basis to step */
 	}
-	int b_op = busy_rate(oi);
 	int r_up = fp_rate(oi + 1);  /* one step less sensitive */
 	int r_dn = fp_rate(oi - 1);  /* frontier, one step more sensitive */
 
-	/* Airtime protection (highest priority): if the operating level defers
-	 * too large a fraction of TX attempts — real traffic included — back off
-	 * to a less sensitive detPeak.  On a congested hilltop most of that busy
-	 * is distant traffic we'd win on capture anyway; deferring for all of it
-	 * just starves our own airtime.  Cap is `set cad.busycap` percent (0 =
-	 * off); only binds on genuinely busy channels. */
+	/* Airtime protection used to live here as the highest-priority rung.  It
+	 * has moved to cadSafetyStep(), which the callers run BEFORE this and
+	 * without the _cad_auto gate — it is a safety, not an optimisation, and
+	 * gating it meant a node whose detPeak could never clear had no way back
+	 * once the operator turned auto off.  What remains here is purely the
+	 * knee-seeking optimiser. */
 	int cap_permille = (int)_cad_busycap_pct * 10;
-	if (cap_permille && _cad_offset < cadLevelMaxEff() && b_op > cap_permille) {
-		_cad_offset++;
-		hwCadSetPeakOffset(_cad_offset);
-		LOG_INF("cad: step up -> offset %d (airtime, busy %d cap %d)",
-			(int)_cad_offset, b_op, cap_permille);
-		return;
-	}
 
 	/* Step UP (less sensitive) when the level above is markedly cleaner —
 	 * we're on the steep part of the curve, below the knee. */
@@ -1793,6 +1801,86 @@ void LoRaRadioBase::cadStaircaseStep()
 	 * plateau — hold. */
 }
 
+bool LoRaRadioBase::cadSafetyStep()
+{
+	/* The airtime-protection rung, run unconditionally — see the constant
+	 * block in radio_common.h for why it is not gated on _cad_auto.
+	 *
+	 * This is the proactive half of the pair.  cadRelaxOnTxStarvation() only
+	 * fires once the node actually has traffic it cannot send, which on a
+	 * silent mesh may be up to flood_advert_interval away (47 h by default);
+	 * this one works off probe statistics, which accumulate whether or not
+	 * there is anything to transmit. */
+	if (_cad_offset >= cadLevelMaxEff()) {
+		return false;
+	}
+
+	int oi = _cad_offset - CAD_LEVEL_MIN;
+	if (oi < 0 || oi >= CAD_NUM_LEVELS) {
+		return false;
+	}
+
+	uint16_t probes = _cad_stats[oi].probes;
+	if (probes == 0) {
+		return false;
+	}
+
+	int b_op = (int)(((uint32_t)_cad_stats[oi].busy * 1000U) / probes);
+	int cap_permille = (int)_cad_busycap_pct * 10;
+
+	/* Unambiguous: this level trips on nearly every probe, so it cannot
+	 * clear for a transmit either.  Acts on few samples and ignores the
+	 * cap. */
+	bool pathological = probes >= CAD_SAFETY_MIN_PROBES &&
+			    b_op >= CAD_SAFETY_PATHOLOGICAL_PERMILLE;
+	/* Marginal: a real airtime-vs-capture tradeoff.  Keeps the original
+	 * evidence bar and honours `cad.busycap 0` as the operator's choice. */
+	bool over_cap = cap_permille && probes >= CAD_STEP_MIN_PROBES &&
+			b_op > cap_permille;
+
+	if (!pathological && !over_cap) {
+		return false;
+	}
+
+	_cad_offset++;
+	hwCadSetPeakOffset(_cad_offset);
+	if (pathological) {
+		LOG_WRN("cad: safety step up -> offset %d (busy %d permille over "
+			"%u probes — detector too sensitive to clear, auto=%d)",
+			(int)_cad_offset, b_op, (unsigned)probes, (int)_cad_auto);
+	} else {
+		LOG_INF("cad: step up -> offset %d (airtime, busy %d cap %d)",
+			(int)_cad_offset, b_op, cap_permille);
+	}
+	return true;
+}
+
+bool LoRaRadioBase::cadRelaxOnTxStarvation()
+{
+	/* Deliberately does NOT consult _cad_auto.  Every other mover of
+	 * _cad_offset is an optimiser and correctly stays out of the way when
+	 * the operator has taken manual control; this one exists precisely for
+	 * the case where manual control produced a node that cannot transmit,
+	 * so honouring `cad.auto off` here would disable the safety exactly
+	 * where it is needed.  It also ignores cad.busycap and the per-level
+	 * probe statistics: 120 warm probes are half an hour away, and the
+	 * caller has already established the harm directly.
+	 *
+	 * One step at a time, never a jump to base: on a genuinely congested
+	 * site the operator's sensitive setting may be almost right, and the
+	 * smallest change that restores transmission is the one to make. */
+	if (_cad_offset >= cadLevelMaxEff()) {
+		return false;
+	}
+
+	_cad_offset++;
+	hwCadSetPeakOffset(_cad_offset);
+	LOG_WRN("cad: TX starvation override -> offset %d (auto=%d) — LBT was "
+		"refusing every transmit",
+		(int)_cad_offset, (int)_cad_auto);
+	return true;
+}
+
 void LoRaRadioBase::cadMaintenance()
 {
 	if (_probe_interval_s == 0) {
@@ -1845,7 +1933,10 @@ void LoRaRadioBase::cadMaintenance()
 		}
 		_cad_pending_level = INT8_MIN;
 
-		if (_cad_auto) {
+		/* Safety first, and outside the _cad_auto gate.  When it acts,
+		 * skip the optimiser this pass — it has just moved the operating
+		 * level and the three-rung window it reads is stale. */
+		if (!cadSafetyStep() && _cad_auto) {
 			cadStaircaseStep();
 		}
 	}
@@ -1983,7 +2074,7 @@ void LoRaRadioBase::cadMaintenance()
 		return;
 	}
 
-	if (_cad_auto) {
+	if (!cadSafetyStep() && _cad_auto) {
 		cadStaircaseStep();
 	}
 }

zephcore/adapters/radio/LoRaRadioBase.h

@@ -154,6 +154,10 @@ public:
 	int8_t cadLevelMinEff();
 	int8_t cadLevelMaxEff();
 	void resetCadStats() override;
+	bool cadRelaxOnTxStarvation() override;
+	/* Airtime-protection / stuck-detector rung, run on every maintenance pass
+	 * regardless of _cad_auto.  Returns true when it moved the offset. */
+	bool cadSafetyStep();
 	int formatCadStatus(char *buf, int cap) override;
 
 protected:

zephcore/adapters/radio/radio_common.h

@@ -154,6 +154,24 @@ static inline uint32_t rssi_settle_delay_us(uint16_t bw_khz)
  * 40% reproduces the old behaviour exactly at cap 25 and holds that same share
  * at every other setting. */
 #define CAD_BUSY_DEFER_HYST_PCT      40   /* descend only if frontier busy <= 60% of cap */
+/* Safety rung thresholds — see LoRaRadioBase::cadSafetyStep().
+ *
+ * The airtime cap above is a SAFETY, not an optimisation, so it runs whether or
+ * not `cad.auto` is on.  A detPeak so sensitive that CAD never clears leaves the
+ * node unable to transmit at all, and the three settings that used to gate the
+ * whole staircase (`cad.auto off`, `cad.busycap 0`, and a level not yet warm to
+ * CAD_STEP_MIN_PROBES) are exactly what a hand-tuning operator turns off — the
+ * CLI help for `set cad.auto` recommends that workflow by name.
+ *
+ * PATHOLOGICAL is the fast path for the unambiguous case.  Proving a MARGINAL
+ * busy rate against the cap needs the full 120 samples, but a level that trips
+ * on essentially every probe needs far fewer to be certain, and it is precisely
+ * the case where waiting half an hour is unacceptable.  It also ignores
+ * `cad.busycap` entirely: a cap of 0 means "do not trade airtime for
+ * sensitivity", which is a policy about a working detector, not permission to
+ * sit mute. */
+#define CAD_SAFETY_MIN_PROBES          20   /* samples before the fast path may act */
+#define CAD_SAFETY_PATHOLOGICAL_PERMILLE 900 /* >=90% busy = detector, not channel */
 #define CAD_PROBE_RSSI_GUARD     7     /* dB above floor = channel visibly busy, skip probe */
 #define CAD_STATS_DECAY_MS       (6UL * 3600UL * 1000UL)  /* halve counters every 6 h */
 /* NOTE: the probe has no retry deadline and no wake of its own.  It runs off

zephcore/include/mesh/Dispatcher.h

@@ -61,6 +61,20 @@ class Dispatcher {
 	uint32_t outbound_expiry, outbound_start, total_air_time, rx_air_time;
 	uint32_t next_tx_time;
 	uint32_t cad_busy_start;
+	/* Separate streak timer for the driver's own LBT verdict.  It cannot
+	 * share cad_busy_start: checkSend() zeroes that unconditionally once the
+	 * pre-TX software gate lets it through, which is every pass on which the
+	 * driver is the one refusing — so the shared variable was reset to 0
+	 * before each refusal could ever accumulate 4 s against it, and the
+	 * escalation below it was unreachable.  Measured on an XIAO ESP32-S3 with
+	 * detPeak forced to its floor: 389 consecutive LBT refusals over 100 s,
+	 * zero warnings, zero error flags, err_events still reading 0. */
+	uint32_t lbt_busy_start;
+	/* Rate limit for the LBT warning.  Separate from lbt_busy_start so that
+	 * warning does not re-arm the streak clock — the starvation override
+	 * below needs the true continuous duration, not the time since the last
+	 * log line. */
+	uint32_t lbt_next_warn;
 	uint32_t tx_budget_ms;
 	uint32_t last_budget_update;
 	uint32_t duty_cycle_window_ms;
@@ -97,6 +111,13 @@ protected:
 	static bool isAdminPacket(const Packet *pkt);
 	virtual uint32_t getCADFailRetryDelay() const;
 	virtual uint32_t getCADFailMaxDuration() const;
+	/* How long the driver's LBT may refuse every transmit before we conclude
+	 * the detector is too sensitive rather than the channel genuinely busy,
+	 * and relax it one step.  Deliberately far longer than
+	 * getCADFailMaxDuration(): a real busy channel nearly always yields SOME
+	 * successful transmit inside a minute, and any success resets the
+	 * streak, so this only fires on a node that is not talking at all. */
+	virtual uint32_t getTxStarvationDuration() const;
 	virtual int getInterferenceThreshold() const { return 0; }
 	virtual uint32_t getDutyCycleWindowMs() const { return 3600000UL; } /* 1h default */
 	/* Adaptive CAD: called when the auto staircase moved the operating

zephcore/include/mesh/Radio.h

@@ -81,6 +81,13 @@ public:
 	virtual uint32_t msUntilNextMaintenance() { return MAINTENANCE_IDLE; }
 	virtual int8_t getCadOffset() const { return 0; }
 	virtual void resetCadStats() {}
+	/* Last-resort unmute: step the CAD detect threshold one notch LESS
+	 * sensitive, ignoring whether adaptive CAD is enabled.  Called by the
+	 * dispatcher only after the driver's LBT has refused every transmit for
+	 * getTxStarvationDuration().  Returns false when already at the radio's
+	 * least sensitive step, which tells the caller the channel is genuinely
+	 * busy (or the radio is broken) rather than the detector mis-tuned. */
+	virtual bool cadRelaxOnTxStarvation() { return false; }
 	/* Writes a human-readable status block; returns chars written (0 = not
 	 * supported by this radio). */
 	virtual int formatCadStatus(char *buf, int cap) {

zephcore/src/Dispatcher.cpp

@@ -31,6 +31,8 @@ Dispatcher::Dispatcher(Radio &radio, MillisecondClock &ms, PacketManager &mgr)
 	total_air_time = rx_air_time = 0;
 	next_tx_time = 0;
 	cad_busy_start = 0;
+	lbt_busy_start = 0;
+	lbt_next_warn = 0;
 	tx_budget_ms = 0;
 	last_budget_update = 0;
 	duty_cycle_window_ms = 0;
@@ -120,6 +122,11 @@ uint32_t Dispatcher::getCADFailMaxDuration() const
 	return 4000; /* ms; ~20 retry attempts before giving up */
 }
 
+uint32_t Dispatcher::getTxStarvationDuration() const
+{
+	return 60000; /* ms; 15 warning periods of not transmitting at all */
+}
+
 void Dispatcher::loop()
 {
 	if (outbound) {
@@ -384,6 +391,16 @@ void Dispatcher::checkSend()
 	int count = _mgr->getOutboundCount(now);
 	if (count == 0) {
 		cad_busy_start = 0;
+		/* Only a genuinely EMPTY queue ends an LBT starvation streak, not
+		 * one that merely has nothing due this instant.  getOutboundCount()
+		 * excludes packets scheduled in the future, and an LBT refusal
+		 * re-queues its packet 100-200 ms ahead — so every checkSend() that
+		 * lands in that retry gap (any RX wake will do) used to reset the
+		 * streak here, and a node refusing every transmit could keep
+		 * restarting the clock instead of ever reaching the escalation. */
+		if (_mgr->getOutboundTotal() == 0) {
+			lbt_busy_start = 0;
+		}
 		return;
 	}
 
@@ -594,15 +611,68 @@ void Dispatcher::checkSend()
 				 * channel is a true reading and walking the radio
 				 * through REST would only add deaf time. Report and
 				 * keep retrying. */
-				if (cad_busy_start == 0) {
-					cad_busy_start = now;
-				} else if (now - cad_busy_start > getCADFailMaxDuration()) {
-					_err_flags |= ERR_EVENT_CAD_TIMEOUT;
-					LOG_WRN("checkSend: LBT has refused TX for %ums "
-						"(len=%d, noise=%d) — channel busy or CAD too sensitive",
-						(unsigned)(now - cad_busy_start), len,
-						_radio->getNoiseFloor());
-					cad_busy_start = now;
+				if (lbt_busy_start == 0) {
+					lbt_busy_start = now;
+					lbt_next_warn = now + getCADFailMaxDuration();
+				} else {
+					uint32_t streak = now - lbt_busy_start;
+
+					if ((int32_t)(now - lbt_next_warn) >= 0) {
+						_err_flags |= ERR_EVENT_CAD_TIMEOUT;
+						LOG_WRN("checkSend: LBT has refused TX for %ums "
+							"(len=%d, noise=%d) — channel busy or CAD too sensitive",
+							(unsigned)streak, len,
+							_radio->getNoiseFloor());
+						lbt_next_warn = now + getCADFailMaxDuration();
+					}
+
+					/* Self-unmute.  A node whose LBT has refused EVERY
+					 * transmit for this long is not looking at a busy
+					 * channel — a busy channel still yields gaps, and any
+					 * success resets this streak.  It is looking at a
+					 * detector tuned too sensitive to ever clear, which
+					 * `set cad.offset -8` on a quiet site produces
+					 * outright.
+					 *
+					 * The adaptive staircase cannot be relied on to undo
+					 * that: it runs only when `cad.auto` is on, only when
+					 * `cad.busycap` is non-zero, and only once the
+					 * operating level has 120 probes behind it — half an
+					 * hour at best.  All three are settings a hand-tuning
+					 * operator turns off, and the CLI help for
+					 * `set cad.auto` recommends exactly that workflow.  A
+					 * repeater on a mast that stops transmitting cannot
+					 * be talked back down either: it still receives and
+					 * still applies an admin `set`, but the reply never
+					 * gets out, so no ordinary app completes the login it
+					 * is waiting on.
+					 *
+					 * So this deliberately overrides `cad.auto` and runs
+					 * regardless of the operator's settings.  The radio
+					 * clamps to its own least-sensitive step and reports
+					 * when it can go no further.
+					 *
+					 * The new offset PERSISTS, like a staircase step:
+					 * maintenanceLoop() notices getCadOffset() moved and
+					 * calls onCadOffsetChanged(), which writes prefs.
+					 * That is the behaviour we want on a mast — a node
+					 * that healed itself must not go mute again on the
+					 * next reboot — and it is what `cad.auto` already
+					 * does to a hand-set offset.  Writes are bounded: each
+					 * step needs another full starvation period, and they
+					 * stop the moment a transmit succeeds. */
+					if (streak > getTxStarvationDuration()) {
+						if (_radio->cadRelaxOnTxStarvation()) {
+							LOG_ERR("checkSend: TX starved %ums — relaxed CAD one step",
+								(unsigned)streak);
+						} else {
+							LOG_ERR("checkSend: TX starved %ums — CAD already at its "
+								"least sensitive step, channel may be genuinely busy",
+								(unsigned)streak);
+						}
+						lbt_busy_start = now;
+						lbt_next_warn = now + getCADFailMaxDuration();
+					}
 				}
 				logTxFail(outbound, outbound->getRawLength());
 				_mgr->queueOutbound(outbound, outbound_priority, futureMillis((int)retry));
@@ -618,6 +688,7 @@ void Dispatcher::checkSend()
 				 * inherits a stale start, reporting a stall that
 				 * already ended. */
 				cad_busy_start = 0;
+				lbt_busy_start = 0;
 			}
 		}
 	}