Added RX duty-cycle watchdog recovery

examples/simple_repeater/MyMesh.cpp

@@ -1076,6 +1076,11 @@ bool MyMesh::setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) {
                      ok ? "" : " unsupported");
   return ok;
 }
+void MyMesh::getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) {
+  *soft = radio_driver.getRxPsWatchdogSoftCount();
+  *hard = radio_driver.getRxPsWatchdogHardCount();
+}
+
 
 #if defined(USE_SX1262) || defined(USE_SX1268)
 void MyMesh::setRxBoostedGain(bool enable) {

examples/simple_repeater/MyMesh.h

@@ -213,6 +213,7 @@ public:
   void dumpLogFile() override;
   void setTxPower(int8_t power_dbm) override;
   bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) override;
+  void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) override;
   void formatNeighborsReply(char *reply) override;
   void removeNeighbor(const uint8_t* pubkey, int key_len) override;
   void formatStatsReply(char *reply) override;

examples/simple_room_server/MyMesh.cpp

@@ -814,6 +814,11 @@ void MyMesh::setTxPower(int8_t power_dbm) {
 bool MyMesh::setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) {
   return radio_driver.setRxPowerSaving(enable, rx_us, sleep_us);
 }
+void MyMesh::getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) {
+  *soft = radio_driver.getRxPsWatchdogSoftCount();
+  *hard = radio_driver.getRxPsWatchdogHardCount();
+}
+
 
 void MyMesh::saveIdentity(const mesh::LocalIdentity &new_id) {
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)

examples/simple_room_server/MyMesh.h

@@ -207,6 +207,7 @@ public:
   void dumpLogFile() override;
   void setTxPower(int8_t power_dbm) override;
   bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) override;
+  void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) override;
 
   void formatNeighborsReply(char *reply) override {
     strcpy(reply, "not supported");

examples/simple_sensor/SensorMesh.cpp

@@ -855,6 +855,11 @@ void SensorMesh::setTxPower(int8_t power_dbm) {
 bool SensorMesh::setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) {
   return radio_driver.setRxPowerSaving(enable, rx_us, sleep_us);
 }
+void SensorMesh::getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) {
+  *soft = radio_driver.getRxPsWatchdogSoftCount();
+  *hard = radio_driver.getRxPsWatchdogHardCount();
+}
+
 
 void SensorMesh::formatStatsReply(char *reply) {
   StatsFormatHelper::formatCoreStats(reply, board, *_ms, _err_flags, _mgr);

examples/simple_sensor/SensorMesh.h

@@ -69,6 +69,7 @@ public:
   void dumpLogFile() override { }
   void setTxPower(int8_t power_dbm) override;
   bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) override;
+  void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) override;
   void formatNeighborsReply(char *reply) override {
     strcpy(reply, "not supported");
   }

src/helpers/CommonCLI.cpp

@@ -1090,6 +1090,10 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep
     ensureRxPowerSavingDefaults(_prefs);
     sprintf(reply, "> %s,%lu,%lu", _prefs->rx_powersaving_enabled ? "on" : "off",
             (unsigned long)_prefs->rx_ps_rx_us, (unsigned long)_prefs->rx_ps_sleep_us);
+  } else if (memcmp(config, "rxps.wd", 7) == 0) {
+    uint32_t wd_soft, wd_hard;
+    _callbacks->getRxPsWatchdogCounts(&wd_soft, &wd_hard);
+    sprintf(reply, "> soft=%lu,hard=%lu", (unsigned long)wd_soft, (unsigned long)wd_hard);
   } else if (memcmp(config, "radio", 5) == 0) {
     char freq[16], bw[16];
     strcpy(freq, StrHelper::ftoa(_prefs->freq));

src/helpers/CommonCLI.h

@@ -136,6 +136,10 @@ public:
   virtual bool setRxPowerSaving(bool enable, uint32_t rx_us, uint32_t sleep_us) {
     return !enable;
   };
+
+  virtual void getRxPsWatchdogCounts(uint32_t* soft, uint32_t* hard) {
+    *soft = 0; *hard = 0;
+  };
 };
 
 class CommonCLI {

src/helpers/radiolib/CustomSX1262Wrapper.h

@@ -13,6 +13,7 @@ public:
   CustomSX1262Wrapper(CustomSX1262& radio, mesh::MainBoard& board) : RadioLibWrapper(radio, board) { }
 
   void setParams(float freq, float bw, uint8_t sf, uint8_t cr) override {
+    cacheParams(freq, bw, sf, cr);
     ((CustomSX1262 *)_radio)->setFrequency(freq);
     ((CustomSX1262 *)_radio)->setSpreadingFactor(sf);
     ((CustomSX1262 *)_radio)->setBandwidth(bw);
@@ -78,6 +79,12 @@ protected:
     _rx_ps_armed = false;
   }
 
+  bool radioDeepInit() override {
+    // std_init() re-runs RadioLib begin(), which starts with a hardware reset
+    // via NRST - the only way out of a hard-locked chip (BUSY stuck high).
+    return ((CustomSX1262 *)_radio)->std_init();
+  }
+
 public:
   void doResetAGC() override { sx126xResetAGC((SX126x *)_radio); }
 

src/helpers/radiolib/RadioLibWrappers.cpp

@@ -48,6 +48,8 @@ uint32_t RadioLibWrapper::getRngSeed() {
 }
 
 void RadioLibWrapper::setTxPower(int8_t dbm) {
+  _cur_dbm = dbm;
+  _dbm_valid = true;
   _radio->setOutputPower(dbm);
 }
 
@@ -84,7 +86,54 @@ void RadioLibWrapper::resetAGC() {
   _floor_sample_sum = 0;
 }
 
+void RadioLibWrapper::rxPsWatchdogCheck() {
+  // don't interfere mid-transmit or with a completed-but-unread packet
+  if ((state & STATE_INT_READY) != 0 || (state & ~STATE_INT_READY) == STATE_TX_WAIT) return;
+
+  unsigned long now = millis();
+  bool tripped = false;
+
+  if (_rx_ps_armed && state == STATE_RX && _wd_stuck_thresh > 0) {
+    bool busy = isChipBusy();
+    if (busy != _wd_last_busy) {
+      _wd_last_busy = busy;
+      _wd_last_transition = now;
+      _wd_stage = 0;    // the sleep/listen wave is present -> radio healthy
+    } else if (now - _wd_last_transition > _wd_stuck_thresh) {
+      tripped = true;   // BUSY frozen: chip fell out of the duty cycle with no IRQ
+    }
+  }
+  if (_startrx_fails >= 3) tripped = true;  // can't even re-arm receive mode
+
+  if (!tripped) return;
+
+  _wd_last_transition = now;   // grace period before the next escalation
+  _startrx_fails = 0;
+
+  if (_wd_stage == 0) {
+    _wd_stage = 1;
+    n_wd_soft++;
+    MESH_DEBUG_PRINTLN("RadioLibWrapper: watchdog: RX duty-cycle stuck, soft re-arm");
+    state = STATE_IDLE;   // next recvRaw() re-arms receive mode
+  } else {
+    _wd_stage = 2;
+    n_wd_hard++;
+    MESH_DEBUG_PRINTLN("RadioLibWrapper: watchdog: still stuck, hard radio reset");
+    if (radioDeepInit()) {
+      _rx_ps_armed = false;   // chip is factory-fresh after NRST
+      _radio->setPacketReceivedAction(setFlag);
+      if (_params_valid) setParams(_cur_freq, _cur_bw, _cur_sf, _cur_cr);
+      if (_dbm_valid) _radio->setOutputPower(_cur_dbm);
+    }
+    state = STATE_IDLE;   // re-arm (rx powersaving settings are kept in members)
+  }
+}
+
 void RadioLibWrapper::loop() {
+  if (_rx_ps_enabled) {
+    rxPsWatchdogCheck();
+  }
+
   if (state == STATE_RX && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) {
     // In RX duty-cycle (powersaving) mode only sample while the chip is in a
     // listen window: during the sleep window the frontend is off, so the RSSI
@@ -111,7 +160,20 @@ void RadioLibWrapper::startRecv() {
   int err = startReceiveMode();
   if (err == RADIOLIB_ERR_NONE) {
     state = STATE_RX;
+    _startrx_fails = 0;
+    if (_rx_ps_armed) {
+      // (re)base the duty-cycle watchdog on the freshly armed cycle
+      _wd_last_busy = isChipBusy();
+      _wd_last_transition = millis();
+      // Longest legitimate silence on the BUSY pin: one full cycle, plus the
+      // extended RX after a (possibly false) preamble detect (2*rx + sleep),
+      // plus a worst-case packet airtime, plus margin for TCXO/transitions.
+      uint32_t rx_ms = _rx_ps_rx_us / 1000, sleep_ms = _rx_ps_sleep_us / 1000;
+      _wd_stuck_thresh = (rx_ms + sleep_ms) + 2 * (2 * rx_ms + sleep_ms)
+                         + getEstAirtimeFor(MAX_TRANS_UNIT) + 1000;
+    }
   } else {
+    if (_startrx_fails < 255) _startrx_fails++;
     MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceiveMode(%d)", err);
   }
 }

src/helpers/radiolib/RadioLibWrappers.h

@@ -25,14 +25,37 @@ protected:
   uint32_t _rx_ps_rx_us;
   uint32_t _rx_ps_sleep_us;
 
+  // RX duty-cycle watchdog: a healthy duty cycle shows a square wave on the
+  // BUSY pin (high in the sleep window / TCXO warmup, low while listening).
+  // If the wave stops, the chip fell out of the cycle without an IRQ.
+  bool _wd_last_busy;
+  uint8_t _wd_stage;                  // 0 = healthy, 1 = soft re-arm done, 2 = hard reset done
+  uint8_t _startrx_fails;             // consecutive startReceiveMode() failures
+  unsigned long _wd_last_transition;  // millis of last BUSY level change
+  unsigned long _wd_stuck_thresh;     // ms without a transition considered stuck
+  uint32_t n_wd_soft, n_wd_hard;
+
+  // last applied radio settings, reapplied after a hard radio reset
+  float _cur_freq, _cur_bw;
+  uint8_t _cur_sf, _cur_cr;
+  int8_t _cur_dbm;
+  bool _params_valid, _dbm_valid;
+
   void idle();
   void startRecv();
+  void rxPsWatchdogCheck();
+  void cacheParams(float freq, float bw, uint8_t sf, uint8_t cr) {
+    _cur_freq = freq; _cur_bw = bw; _cur_sf = sf; _cur_cr = cr; _params_valid = true;
+  }
   virtual int startReceiveMode();
   virtual void stopReceiveDutyCycle();
   virtual bool isPacketReady();
   // true while the chip cannot service SPI (RX duty-cycle sleep window, or
   // briefly while processing a command); radios expose this via the BUSY pin
   virtual bool isChipBusy() { return false; }
+  // full radio recovery: hardware reset (NRST) + re-init to boot defaults;
+  // returns false if unsupported. Caller reapplies cached runtime params.
+  virtual bool radioDeepInit() { return false; }
   float packetScoreInt(float snr, int sf, int packet_len);
   virtual bool isReceivingPacket() =0;
   virtual void doResetAGC();
@@ -40,7 +63,9 @@ protected:
 public:
   RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board)
       : _radio(&radio), _board(&board), _preamble_sf(0), _rx_ps_enabled(false), _rx_ps_armed(false),
-        _rx_ps_rx_us(RX_PS_FALLBACK_RX_US), _rx_ps_sleep_us(RX_PS_FALLBACK_SLEEP_US) { n_recv = n_sent = n_recv_errors = 0; }
+        _rx_ps_rx_us(RX_PS_FALLBACK_RX_US), _rx_ps_sleep_us(RX_PS_FALLBACK_SLEEP_US),
+        _wd_last_busy(false), _wd_stage(0), _startrx_fails(0), _wd_last_transition(0), _wd_stuck_thresh(0),
+        _params_valid(false), _dbm_valid(false) { n_recv = n_sent = n_recv_errors = n_wd_soft = n_wd_hard = 0; }
 
   void begin() override;
   virtual void powerOff() { _radio->sleep(); }
@@ -79,6 +104,8 @@ public:
   uint32_t getPacketsRecv() const { return n_recv; }
   uint32_t getPacketsRecvErrors() const { return n_recv_errors; }
   uint32_t getPacketsSent() const { return n_sent; }
+  uint32_t getRxPsWatchdogSoftCount() const { return n_wd_soft; }
+  uint32_t getRxPsWatchdogHardCount() const { return n_wd_hard; }
   void resetStats() { n_recv = n_sent = n_recv_errors = 0; }
 
   virtual float getLastRSSI() const override;