diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index fdd93be7..d73c6632 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1076,6 +1076,11 @@ bool MyMesh::setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) { ok ? "" : " unsupported"); return ok; } +void MyMesh::getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) { + *soft = radio_driver.getRxPsWatchdogSoftCount(); + *hard = radio_driver.getRxPsWatchdogHardCount(); +} + #if defined(USE_SX1262) || defined(USE_SX1268) void MyMesh::setRxBoostedGain(bool enable) { diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index a37c8f74..a3fabc5b 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -213,6 +213,7 @@ public: void dumpLogFile() override; void setTxPower(int8_t power_dbm) override; bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) override; + void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) override; void formatNeighborsReply(char *reply) override; void removeNeighbor(const uint8_t* pubkey, int key_len) override; void formatStatsReply(char *reply) override; diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 9cd14647..0eed525f 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -814,6 +814,11 @@ void MyMesh::setTxPower(int8_t power_dbm) { bool MyMesh::setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) { return radio_driver.setRxPowerSaving(enable, rx_us, sleep_us); } +void MyMesh::getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) { + *soft = radio_driver.getRxPsWatchdogSoftCount(); + *hard = radio_driver.getRxPsWatchdogHardCount(); +} + void MyMesh::saveIdentity(const mesh::LocalIdentity &new_id) { #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 6205f31a..ff4b951a 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -207,6 +207,7 @@ public: void dumpLogFile() override; void setTxPower(int8_t power_dbm) override; bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) override; + void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) override; void formatNeighborsReply(char *reply) override { strcpy(reply, "not supported"); diff --git a/examples/simple_sensor/SensorMesh.cpp b/examples/simple_sensor/SensorMesh.cpp index 7dc256cb..2cb120d0 100644 --- a/examples/simple_sensor/SensorMesh.cpp +++ b/examples/simple_sensor/SensorMesh.cpp @@ -855,6 +855,11 @@ void SensorMesh::setTxPower(int8_t power_dbm) { bool SensorMesh::setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) { return radio_driver.setRxPowerSaving(enable, rx_us, sleep_us); } +void SensorMesh::getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) { + *soft = radio_driver.getRxPsWatchdogSoftCount(); + *hard = radio_driver.getRxPsWatchdogHardCount(); +} + void SensorMesh::formatStatsReply(char *reply) { StatsFormatHelper::formatCoreStats(reply, board, *_ms, _err_flags, _mgr); diff --git a/examples/simple_sensor/SensorMesh.h b/examples/simple_sensor/SensorMesh.h index e110eaca..e9c9f485 100644 --- a/examples/simple_sensor/SensorMesh.h +++ b/examples/simple_sensor/SensorMesh.h @@ -69,6 +69,7 @@ public: void dumpLogFile() override { } void setTxPower(int8_t power_dbm) override; bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) override; + void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) override; void formatNeighborsReply(char *reply) override { strcpy(reply, "not supported"); } diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 83eb2066..4b094343 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -1090,6 +1090,10 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep ensureRxPowerSavingDefaults(_prefs); sprintf(reply, "> %s,%lu,%lu", _prefs->rx_powersaving_enabled ? "on" : "off", (unsigned long)_prefs->rx_ps_rx_us, (unsigned long)_prefs->rx_ps_sleep_us); + } else if (memcmp(config, "rxps.wd", 7) == 0) { + uint32_t wd_soft, wd_hard; + _callbacks->getRxPsWatchdogCounts(&wd_soft, &wd_hard); + sprintf(reply, "> soft=%lu,hard=%lu", (unsigned long)wd_soft, (unsigned long)wd_hard); } else if (memcmp(config, "radio", 5) == 0) { char freq[16], bw[16]; strcpy(freq, StrHelper::ftoa(_prefs->freq)); diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 46d55ccc..6ec5aa01 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -136,6 +136,10 @@ public: virtual bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) { return !enable; }; + + virtual void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) { + *soft = 0; *hard = 0; + }; }; class CommonCLI { diff --git a/src/helpers/radiolib/CustomSX1262Wrapper.h b/src/helpers/radiolib/CustomSX1262Wrapper.h index 585ba4a9..b1b9ce57 100644 --- a/src/helpers/radiolib/CustomSX1262Wrapper.h +++ b/src/helpers/radiolib/CustomSX1262Wrapper.h @@ -13,6 +13,7 @@ public: CustomSX1262Wrapper(CustomSX1262& radio, mesh::MainBoard& board) : RadioLibWrapper(radio, board) { } void setParams(float freq, float bw, uint8_t sf, uint8_t cr) override { + cacheParams(freq, bw, sf, cr); ((CustomSX1262 *)_radio)->setFrequency(freq); ((CustomSX1262 *)_radio)->setSpreadingFactor(sf); ((CustomSX1262 *)_radio)->setBandwidth(bw); @@ -78,6 +79,12 @@ protected: _rx_ps_armed = false; } + bool radioDeepInit() override { + // std_init() re-runs RadioLib begin(), which starts with a hardware reset + // via NRST - the only way out of a hard-locked chip (BUSY stuck high). + return ((CustomSX1262 *)_radio)->std_init(); + } + public: void doResetAGC() override { sx126xResetAGC((SX126x *)_radio); } diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index f93bf350..203fe163 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -48,6 +48,8 @@ uint32_t RadioLibWrapper::getRngSeed() { } void RadioLibWrapper::setTxPower(int8_t dbm) { + _cur_dbm = dbm; + _dbm_valid = true; _radio->setOutputPower(dbm); } @@ -84,7 +86,54 @@ void RadioLibWrapper::resetAGC() { _floor_sample_sum = 0; } +void RadioLibWrapper::rxPsWatchdogCheck() { + // don't interfere mid-transmit or with a completed-but-unread packet + if ((state & STATE_INT_READY) != 0 || (state & ~STATE_INT_READY) == STATE_TX_WAIT) return; + + unsigned long now = millis(); + bool tripped = false; + + if (_rx_ps_armed && state == STATE_RX && _wd_stuck_thresh > 0) { + bool busy = isChipBusy(); + if (busy != _wd_last_busy) { + _wd_last_busy = busy; + _wd_last_transition = now; + _wd_stage = 0; // the sleep/listen wave is present -> radio healthy + } else if (now - _wd_last_transition > _wd_stuck_thresh) { + tripped = true; // BUSY frozen: chip fell out of the duty cycle with no IRQ + } + } + if (_startrx_fails >= 3) tripped = true; // can't even re-arm receive mode + + if (!tripped) return; + + _wd_last_transition = now; // grace period before the next escalation + _startrx_fails = 0; + + if (_wd_stage == 0) { + _wd_stage = 1; + n_wd_soft++; + MESH_DEBUG_PRINTLN("RadioLibWrapper: watchdog: RX duty-cycle stuck, soft re-arm"); + state = STATE_IDLE; // next recvRaw() re-arms receive mode + } else { + _wd_stage = 2; + n_wd_hard++; + MESH_DEBUG_PRINTLN("RadioLibWrapper: watchdog: still stuck, hard radio reset"); + if (radioDeepInit()) { + _rx_ps_armed = false; // chip is factory-fresh after NRST + _radio->setPacketReceivedAction(setFlag); + if (_params_valid) setParams(_cur_freq, _cur_bw, _cur_sf, _cur_cr); + if (_dbm_valid) _radio->setOutputPower(_cur_dbm); + } + state = STATE_IDLE; // re-arm (rx powersaving settings are kept in members) + } +} + void RadioLibWrapper::loop() { + if (_rx_ps_enabled) { + rxPsWatchdogCheck(); + } + if (state == STATE_RX && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) { // In RX duty-cycle (powersaving) mode only sample while the chip is in a // listen window: during the sleep window the frontend is off, so the RSSI @@ -111,7 +160,20 @@ void RadioLibWrapper::startRecv() { int err = startReceiveMode(); if (err == RADIOLIB_ERR_NONE) { state = STATE_RX; + _startrx_fails = 0; + if (_rx_ps_armed) { + // (re)base the duty-cycle watchdog on the freshly armed cycle + _wd_last_busy = isChipBusy(); + _wd_last_transition = millis(); + // Longest legitimate silence on the BUSY pin: one full cycle, plus the + // extended RX after a (possibly false) preamble detect (2*rx + sleep), + // plus a worst-case packet airtime, plus margin for TCXO/transitions. + uint32_t rx_ms = _rx_ps_rx_us / 1000, sleep_ms = _rx_ps_sleep_us / 1000; + _wd_stuck_thresh = (rx_ms + sleep_ms) + 2 * (2 * rx_ms + sleep_ms) + + getEstAirtimeFor(MAX_TRANS_UNIT) + 1000; + } } else { + if (_startrx_fails < 255) _startrx_fails++; MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceiveMode(%d)", err); } } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index a4236982..6ecf26c9 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -25,14 +25,37 @@ protected: uint32_t _rx_ps_rx_us; uint32_t _rx_ps_sleep_us; + // RX duty-cycle watchdog: a healthy duty cycle shows a square wave on the + // BUSY pin (high in the sleep window / TCXO warmup, low while listening). + // If the wave stops, the chip fell out of the cycle without an IRQ. + bool _wd_last_busy; + uint8_t _wd_stage; // 0 = healthy, 1 = soft re-arm done, 2 = hard reset done + uint8_t _startrx_fails; // consecutive startReceiveMode() failures + unsigned long _wd_last_transition; // millis of last BUSY level change + unsigned long _wd_stuck_thresh; // ms without a transition considered stuck + uint32_t n_wd_soft, n_wd_hard; + + // last applied radio settings, reapplied after a hard radio reset + float _cur_freq, _cur_bw; + uint8_t _cur_sf, _cur_cr; + int8_t _cur_dbm; + bool _params_valid, _dbm_valid; + void idle(); void startRecv(); + void rxPsWatchdogCheck(); + void cacheParams(float freq, float bw, uint8_t sf, uint8_t cr) { + _cur_freq = freq; _cur_bw = bw; _cur_sf = sf; _cur_cr = cr; _params_valid = true; + } virtual int startReceiveMode(); virtual void stopReceiveDutyCycle(); virtual bool isPacketReady(); // true while the chip cannot service SPI (RX duty-cycle sleep window, or // briefly while processing a command); radios expose this via the BUSY pin virtual bool isChipBusy() { return false; } + // full radio recovery: hardware reset (NRST) + re-init to boot defaults; + // returns false if unsupported. Caller reapplies cached runtime params. + virtual bool radioDeepInit() { return false; } float packetScoreInt(float snr, int sf, int packet_len); virtual bool isReceivingPacket() =0; virtual void doResetAGC(); @@ -40,7 +63,9 @@ protected: public: RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board), _preamble_sf(0), _rx_ps_enabled(false), _rx_ps_armed(false), - _rx_ps_rx_us(RX_PS_FALLBACK_RX_US), _rx_ps_sleep_us(RX_PS_FALLBACK_SLEEP_US) { n_recv = n_sent = n_recv_errors = 0; } + _rx_ps_rx_us(RX_PS_FALLBACK_RX_US), _rx_ps_sleep_us(RX_PS_FALLBACK_SLEEP_US), + _wd_last_busy(false), _wd_stage(0), _startrx_fails(0), _wd_last_transition(0), _wd_stuck_thresh(0), + _params_valid(false), _dbm_valid(false) { n_recv = n_sent = n_recv_errors = n_wd_soft = n_wd_hard = 0; } void begin() override; virtual void powerOff() { _radio->sleep(); } @@ -79,6 +104,8 @@ public: uint32_t getPacketsRecv() const { return n_recv; } uint32_t getPacketsRecvErrors() const { return n_recv_errors; } uint32_t getPacketsSent() const { return n_sent; } + uint32_t getRxPsWatchdogSoftCount() const { return n_wd_soft; } + uint32_t getRxPsWatchdogHardCount() const { return n_wd_hard; } void resetStats() { n_recv = n_sent = n_recv_errors = 0; } virtual float getLastRSSI() const override;