From bdf734820850dcb10046110301c2f89550505d2d Mon Sep 17 00:00:00 2001 From: liquidraver <504870+liquidraver@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:12:41 +0200 Subject: [PATCH] CAD offset safety net --- zephcore/adapters/radio/LoRaRadioBase.cpp | 123 +++++++++++++++++++--- zephcore/adapters/radio/LoRaRadioBase.h | 4 + zephcore/adapters/radio/radio_common.h | 18 ++++ zephcore/include/mesh/Dispatcher.h | 21 ++++ zephcore/include/mesh/Radio.h | 7 ++ zephcore/src/Dispatcher.cpp | 89 ++++++++++++++-- 6 files changed, 237 insertions(+), 25 deletions(-) diff --git a/zephcore/adapters/radio/LoRaRadioBase.cpp b/zephcore/adapters/radio/LoRaRadioBase.cpp index 0f353e4..5626592 100644 --- a/zephcore/adapters/radio/LoRaRadioBase.cpp +++ b/zephcore/adapters/radio/LoRaRadioBase.cpp @@ -1683,6 +1683,22 @@ int8_t LoRaRadioBase::pickCadProbeLevel() _cad_probe_rr++; if (!_cad_auto) { + /* The sweep window is ABSOLUTE, so an operating offset parked + * outside it is never probed at all — measured: 49 minutes at + * offset -8 produced 0 probes at -8, which left cadSafetyStep() + * with no evidence and made it a no-op in precisely the + * configuration it exists for (auto off + an offset that cannot + * clear). + * + * When the operator has parked outside the window, probe where + * they actually are instead: the swept curve does not contain + * their operating point, so it cannot inform the hand-tuning it + * was built for either. Inside the window the sweep already + * covers the operating level and is left exactly as it was. */ + if (_cad_offset < CAD_SWEEP_MIN || _cad_offset > CAD_SWEEP_MAX) { + return _cad_offset; + } + /* Dry-run: even sweep across the observation window so the * user sees the whole FP-vs-detPeak curve in `get cad`. */ int span = CAD_SWEEP_MAX - CAD_SWEEP_MIN + 1; @@ -1739,24 +1755,16 @@ void LoRaRadioBase::cadStaircaseStep() if (r_op < 0) { return; /* operating level not warm yet — no basis to step */ } - int b_op = busy_rate(oi); int r_up = fp_rate(oi + 1); /* one step less sensitive */ int r_dn = fp_rate(oi - 1); /* frontier, one step more sensitive */ - /* Airtime protection (highest priority): if the operating level defers - * too large a fraction of TX attempts — real traffic included — back off - * to a less sensitive detPeak. On a congested hilltop most of that busy - * is distant traffic we'd win on capture anyway; deferring for all of it - * just starves our own airtime. Cap is `set cad.busycap` percent (0 = - * off); only binds on genuinely busy channels. */ + /* Airtime protection used to live here as the highest-priority rung. It + * has moved to cadSafetyStep(), which the callers run BEFORE this and + * without the _cad_auto gate — it is a safety, not an optimisation, and + * gating it meant a node whose detPeak could never clear had no way back + * once the operator turned auto off. What remains here is purely the + * knee-seeking optimiser. */ int cap_permille = (int)_cad_busycap_pct * 10; - if (cap_permille && _cad_offset < cadLevelMaxEff() && b_op > cap_permille) { - _cad_offset++; - hwCadSetPeakOffset(_cad_offset); - LOG_INF("cad: step up -> offset %d (airtime, busy %d cap %d)", - (int)_cad_offset, b_op, cap_permille); - return; - } /* Step UP (less sensitive) when the level above is markedly cleaner — * we're on the steep part of the curve, below the knee. */ @@ -1793,6 +1801,86 @@ void LoRaRadioBase::cadStaircaseStep() * plateau — hold. */ } +bool LoRaRadioBase::cadSafetyStep() +{ + /* The airtime-protection rung, run unconditionally — see the constant + * block in radio_common.h for why it is not gated on _cad_auto. + * + * This is the proactive half of the pair. cadRelaxOnTxStarvation() only + * fires once the node actually has traffic it cannot send, which on a + * silent mesh may be up to flood_advert_interval away (47 h by default); + * this one works off probe statistics, which accumulate whether or not + * there is anything to transmit. */ + if (_cad_offset >= cadLevelMaxEff()) { + return false; + } + + int oi = _cad_offset - CAD_LEVEL_MIN; + if (oi < 0 || oi >= CAD_NUM_LEVELS) { + return false; + } + + uint16_t probes = _cad_stats[oi].probes; + if (probes == 0) { + return false; + } + + int b_op = (int)(((uint32_t)_cad_stats[oi].busy * 1000U) / probes); + int cap_permille = (int)_cad_busycap_pct * 10; + + /* Unambiguous: this level trips on nearly every probe, so it cannot + * clear for a transmit either. Acts on few samples and ignores the + * cap. */ + bool pathological = probes >= CAD_SAFETY_MIN_PROBES && + b_op >= CAD_SAFETY_PATHOLOGICAL_PERMILLE; + /* Marginal: a real airtime-vs-capture tradeoff. Keeps the original + * evidence bar and honours `cad.busycap 0` as the operator's choice. */ + bool over_cap = cap_permille && probes >= CAD_STEP_MIN_PROBES && + b_op > cap_permille; + + if (!pathological && !over_cap) { + return false; + } + + _cad_offset++; + hwCadSetPeakOffset(_cad_offset); + if (pathological) { + LOG_WRN("cad: safety step up -> offset %d (busy %d permille over " + "%u probes — detector too sensitive to clear, auto=%d)", + (int)_cad_offset, b_op, (unsigned)probes, (int)_cad_auto); + } else { + LOG_INF("cad: step up -> offset %d (airtime, busy %d cap %d)", + (int)_cad_offset, b_op, cap_permille); + } + return true; +} + +bool LoRaRadioBase::cadRelaxOnTxStarvation() +{ + /* Deliberately does NOT consult _cad_auto. Every other mover of + * _cad_offset is an optimiser and correctly stays out of the way when + * the operator has taken manual control; this one exists precisely for + * the case where manual control produced a node that cannot transmit, + * so honouring `cad.auto off` here would disable the safety exactly + * where it is needed. It also ignores cad.busycap and the per-level + * probe statistics: 120 warm probes are half an hour away, and the + * caller has already established the harm directly. + * + * One step at a time, never a jump to base: on a genuinely congested + * site the operator's sensitive setting may be almost right, and the + * smallest change that restores transmission is the one to make. */ + if (_cad_offset >= cadLevelMaxEff()) { + return false; + } + + _cad_offset++; + hwCadSetPeakOffset(_cad_offset); + LOG_WRN("cad: TX starvation override -> offset %d (auto=%d) — LBT was " + "refusing every transmit", + (int)_cad_offset, (int)_cad_auto); + return true; +} + void LoRaRadioBase::cadMaintenance() { if (_probe_interval_s == 0) { @@ -1845,7 +1933,10 @@ void LoRaRadioBase::cadMaintenance() } _cad_pending_level = INT8_MIN; - if (_cad_auto) { + /* Safety first, and outside the _cad_auto gate. When it acts, + * skip the optimiser this pass — it has just moved the operating + * level and the three-rung window it reads is stale. */ + if (!cadSafetyStep() && _cad_auto) { cadStaircaseStep(); } } @@ -1983,7 +2074,7 @@ void LoRaRadioBase::cadMaintenance() return; } - if (_cad_auto) { + if (!cadSafetyStep() && _cad_auto) { cadStaircaseStep(); } } diff --git a/zephcore/adapters/radio/LoRaRadioBase.h b/zephcore/adapters/radio/LoRaRadioBase.h index 48cf367..637a2bd 100644 --- a/zephcore/adapters/radio/LoRaRadioBase.h +++ b/zephcore/adapters/radio/LoRaRadioBase.h @@ -154,6 +154,10 @@ public: int8_t cadLevelMinEff(); int8_t cadLevelMaxEff(); void resetCadStats() override; + bool cadRelaxOnTxStarvation() override; + /* Airtime-protection / stuck-detector rung, run on every maintenance pass + * regardless of _cad_auto. Returns true when it moved the offset. */ + bool cadSafetyStep(); int formatCadStatus(char *buf, int cap) override; protected: diff --git a/zephcore/adapters/radio/radio_common.h b/zephcore/adapters/radio/radio_common.h index a42d1c8..3604491 100644 --- a/zephcore/adapters/radio/radio_common.h +++ b/zephcore/adapters/radio/radio_common.h @@ -154,6 +154,24 @@ static inline uint32_t rssi_settle_delay_us(uint16_t bw_khz) * 40% reproduces the old behaviour exactly at cap 25 and holds that same share * at every other setting. */ #define CAD_BUSY_DEFER_HYST_PCT 40 /* descend only if frontier busy <= 60% of cap */ +/* Safety rung thresholds — see LoRaRadioBase::cadSafetyStep(). + * + * The airtime cap above is a SAFETY, not an optimisation, so it runs whether or + * not `cad.auto` is on. A detPeak so sensitive that CAD never clears leaves the + * node unable to transmit at all, and the three settings that used to gate the + * whole staircase (`cad.auto off`, `cad.busycap 0`, and a level not yet warm to + * CAD_STEP_MIN_PROBES) are exactly what a hand-tuning operator turns off — the + * CLI help for `set cad.auto` recommends that workflow by name. + * + * PATHOLOGICAL is the fast path for the unambiguous case. Proving a MARGINAL + * busy rate against the cap needs the full 120 samples, but a level that trips + * on essentially every probe needs far fewer to be certain, and it is precisely + * the case where waiting half an hour is unacceptable. It also ignores + * `cad.busycap` entirely: a cap of 0 means "do not trade airtime for + * sensitivity", which is a policy about a working detector, not permission to + * sit mute. */ +#define CAD_SAFETY_MIN_PROBES 20 /* samples before the fast path may act */ +#define CAD_SAFETY_PATHOLOGICAL_PERMILLE 900 /* >=90% busy = detector, not channel */ #define CAD_PROBE_RSSI_GUARD 7 /* dB above floor = channel visibly busy, skip probe */ #define CAD_STATS_DECAY_MS (6UL * 3600UL * 1000UL) /* halve counters every 6 h */ /* NOTE: the probe has no retry deadline and no wake of its own. It runs off diff --git a/zephcore/include/mesh/Dispatcher.h b/zephcore/include/mesh/Dispatcher.h index 1624401..1fb7247 100644 --- a/zephcore/include/mesh/Dispatcher.h +++ b/zephcore/include/mesh/Dispatcher.h @@ -61,6 +61,20 @@ class Dispatcher { uint32_t outbound_expiry, outbound_start, total_air_time, rx_air_time; uint32_t next_tx_time; uint32_t cad_busy_start; + /* Separate streak timer for the driver's own LBT verdict. It cannot + * share cad_busy_start: checkSend() zeroes that unconditionally once the + * pre-TX software gate lets it through, which is every pass on which the + * driver is the one refusing — so the shared variable was reset to 0 + * before each refusal could ever accumulate 4 s against it, and the + * escalation below it was unreachable. Measured on an XIAO ESP32-S3 with + * detPeak forced to its floor: 389 consecutive LBT refusals over 100 s, + * zero warnings, zero error flags, err_events still reading 0. */ + uint32_t lbt_busy_start; + /* Rate limit for the LBT warning. Separate from lbt_busy_start so that + * warning does not re-arm the streak clock — the starvation override + * below needs the true continuous duration, not the time since the last + * log line. */ + uint32_t lbt_next_warn; uint32_t tx_budget_ms; uint32_t last_budget_update; uint32_t duty_cycle_window_ms; @@ -97,6 +111,13 @@ protected: static bool isAdminPacket(const Packet *pkt); virtual uint32_t getCADFailRetryDelay() const; virtual uint32_t getCADFailMaxDuration() const; + /* How long the driver's LBT may refuse every transmit before we conclude + * the detector is too sensitive rather than the channel genuinely busy, + * and relax it one step. Deliberately far longer than + * getCADFailMaxDuration(): a real busy channel nearly always yields SOME + * successful transmit inside a minute, and any success resets the + * streak, so this only fires on a node that is not talking at all. */ + virtual uint32_t getTxStarvationDuration() const; virtual int getInterferenceThreshold() const { return 0; } virtual uint32_t getDutyCycleWindowMs() const { return 3600000UL; } /* 1h default */ /* Adaptive CAD: called when the auto staircase moved the operating diff --git a/zephcore/include/mesh/Radio.h b/zephcore/include/mesh/Radio.h index f2d4e8b..a5587ed 100644 --- a/zephcore/include/mesh/Radio.h +++ b/zephcore/include/mesh/Radio.h @@ -81,6 +81,13 @@ public: virtual uint32_t msUntilNextMaintenance() { return MAINTENANCE_IDLE; } virtual int8_t getCadOffset() const { return 0; } virtual void resetCadStats() {} + /* Last-resort unmute: step the CAD detect threshold one notch LESS + * sensitive, ignoring whether adaptive CAD is enabled. Called by the + * dispatcher only after the driver's LBT has refused every transmit for + * getTxStarvationDuration(). Returns false when already at the radio's + * least sensitive step, which tells the caller the channel is genuinely + * busy (or the radio is broken) rather than the detector mis-tuned. */ + virtual bool cadRelaxOnTxStarvation() { return false; } /* Writes a human-readable status block; returns chars written (0 = not * supported by this radio). */ virtual int formatCadStatus(char *buf, int cap) { diff --git a/zephcore/src/Dispatcher.cpp b/zephcore/src/Dispatcher.cpp index bab57d9..6650f62 100644 --- a/zephcore/src/Dispatcher.cpp +++ b/zephcore/src/Dispatcher.cpp @@ -31,6 +31,8 @@ Dispatcher::Dispatcher(Radio &radio, MillisecondClock &ms, PacketManager &mgr) total_air_time = rx_air_time = 0; next_tx_time = 0; cad_busy_start = 0; + lbt_busy_start = 0; + lbt_next_warn = 0; tx_budget_ms = 0; last_budget_update = 0; duty_cycle_window_ms = 0; @@ -120,6 +122,11 @@ uint32_t Dispatcher::getCADFailMaxDuration() const return 4000; /* ms; ~20 retry attempts before giving up */ } +uint32_t Dispatcher::getTxStarvationDuration() const +{ + return 60000; /* ms; 15 warning periods of not transmitting at all */ +} + void Dispatcher::loop() { if (outbound) { @@ -384,6 +391,16 @@ void Dispatcher::checkSend() int count = _mgr->getOutboundCount(now); if (count == 0) { cad_busy_start = 0; + /* Only a genuinely EMPTY queue ends an LBT starvation streak, not + * one that merely has nothing due this instant. getOutboundCount() + * excludes packets scheduled in the future, and an LBT refusal + * re-queues its packet 100-200 ms ahead — so every checkSend() that + * lands in that retry gap (any RX wake will do) used to reset the + * streak here, and a node refusing every transmit could keep + * restarting the clock instead of ever reaching the escalation. */ + if (_mgr->getOutboundTotal() == 0) { + lbt_busy_start = 0; + } return; } @@ -594,15 +611,68 @@ void Dispatcher::checkSend() * channel is a true reading and walking the radio * through REST would only add deaf time. Report and * keep retrying. */ - if (cad_busy_start == 0) { - cad_busy_start = now; - } else if (now - cad_busy_start > getCADFailMaxDuration()) { - _err_flags |= ERR_EVENT_CAD_TIMEOUT; - LOG_WRN("checkSend: LBT has refused TX for %ums " - "(len=%d, noise=%d) — channel busy or CAD too sensitive", - (unsigned)(now - cad_busy_start), len, - _radio->getNoiseFloor()); - cad_busy_start = now; + if (lbt_busy_start == 0) { + lbt_busy_start = now; + lbt_next_warn = now + getCADFailMaxDuration(); + } else { + uint32_t streak = now - lbt_busy_start; + + if ((int32_t)(now - lbt_next_warn) >= 0) { + _err_flags |= ERR_EVENT_CAD_TIMEOUT; + LOG_WRN("checkSend: LBT has refused TX for %ums " + "(len=%d, noise=%d) — channel busy or CAD too sensitive", + (unsigned)streak, len, + _radio->getNoiseFloor()); + lbt_next_warn = now + getCADFailMaxDuration(); + } + + /* Self-unmute. A node whose LBT has refused EVERY + * transmit for this long is not looking at a busy + * channel — a busy channel still yields gaps, and any + * success resets this streak. It is looking at a + * detector tuned too sensitive to ever clear, which + * `set cad.offset -8` on a quiet site produces + * outright. + * + * The adaptive staircase cannot be relied on to undo + * that: it runs only when `cad.auto` is on, only when + * `cad.busycap` is non-zero, and only once the + * operating level has 120 probes behind it — half an + * hour at best. All three are settings a hand-tuning + * operator turns off, and the CLI help for + * `set cad.auto` recommends exactly that workflow. A + * repeater on a mast that stops transmitting cannot + * be talked back down either: it still receives and + * still applies an admin `set`, but the reply never + * gets out, so no ordinary app completes the login it + * is waiting on. + * + * So this deliberately overrides `cad.auto` and runs + * regardless of the operator's settings. The radio + * clamps to its own least-sensitive step and reports + * when it can go no further. + * + * The new offset PERSISTS, like a staircase step: + * maintenanceLoop() notices getCadOffset() moved and + * calls onCadOffsetChanged(), which writes prefs. + * That is the behaviour we want on a mast — a node + * that healed itself must not go mute again on the + * next reboot — and it is what `cad.auto` already + * does to a hand-set offset. Writes are bounded: each + * step needs another full starvation period, and they + * stop the moment a transmit succeeds. */ + if (streak > getTxStarvationDuration()) { + if (_radio->cadRelaxOnTxStarvation()) { + LOG_ERR("checkSend: TX starved %ums — relaxed CAD one step", + (unsigned)streak); + } else { + LOG_ERR("checkSend: TX starved %ums — CAD already at its " + "least sensitive step, channel may be genuinely busy", + (unsigned)streak); + } + lbt_busy_start = now; + lbt_next_warn = now + getCADFailMaxDuration(); + } } logTxFail(outbound, outbound->getRawLength()); _mgr->queueOutbound(outbound, outbound_priority, futureMillis((int)retry)); @@ -618,6 +688,7 @@ void Dispatcher::checkSend() * inherits a stale start, reporting a stall that * already ended. */ cad_busy_start = 0; + lbt_busy_start = 0; } } }