Refine trace and group data retries

src/Dispatcher.cpp

@@ -393,6 +393,8 @@ void Dispatcher::processRecvPacket(Packet* pkt) {
     if (!queueOutboundPacket(pkt, priority, _delay)) {
       onSendFail(pkt);
       releasePacket(pkt);
+    } else if (pkt->isRouteDirect() && pkt->getPayloadType() == PAYLOAD_TYPE_TRACE) {
+      onTracePacketQueuedForSend(pkt);
     }
   }
 }
@@ -554,6 +556,9 @@ bool Dispatcher::sendPacket(Packet* packet, uint8_t priority, uint32_t delay_mil
     releasePacket(packet);
     return false;
   }
+  if (packet->isRouteDirect() && packet->getPayloadType() == PAYLOAD_TYPE_TRACE) {
+    onTracePacketQueuedForSend(packet);
+  }
   return true;
 }
 

src/Dispatcher.h

@@ -232,6 +232,7 @@ protected:
   virtual void logRx(Packet* packet, int len, float score) { }   // hooks for custom logging
   virtual void logTx(Packet* packet, int len) { }
   virtual void logTxFail(Packet* packet, int len) { }
+  virtual void onTracePacketQueuedForSend(Packet* packet) { }
   virtual void onSendComplete(Packet* packet) { }
   virtual void onSendFail(Packet* packet) { }
   virtual const char* getLogDateTime() { return ""; }

src/Mesh.cpp

@@ -161,6 +161,7 @@ void Mesh::begin() {
     _direct_retries[i].retry_delay = 0;
     _direct_retries[i].retry_attempts_sent = 0;
     memset(_direct_retries[i].retry_key, 0, sizeof(_direct_retries[i].retry_key));
+    memset(_direct_retries[i].trace_replacement_key, 0, sizeof(_direct_retries[i].trace_replacement_key));
     memset(_direct_retries[i].next_hop_hash, 0, sizeof(_direct_retries[i].next_hop_hash));
     _direct_retries[i].next_hop_hash_len = 0;
     _direct_retries[i].payload_type = 0;
@@ -359,7 +360,7 @@ uint8_t Mesh::getDirectRetryPacketAirtimeFactor(const Packet* packet) const {
 
   uint8_t payload_type = packet->getPayloadType();
   if (payload_type == PAYLOAD_TYPE_TRACE || payload_type == PAYLOAD_TYPE_ANON_REQ) {
-    return 4;
+    return 3;
   }
   if (payload_type == PAYLOAD_TYPE_TXT_MSG) {
     return 7;
@@ -399,6 +400,18 @@ uint8_t Mesh::getFloodRetryMaxPathLength(const Packet* packet) const {
   (void)packet;
   return FLOOD_RETRY_MAX_PATH_DEFAULT;
 }
+uint8_t Mesh::applyGroupDataFloodRetryPathGate(const Packet* packet,
+                                               uint8_t general_gate,
+                                               uint8_t group_data_gate) {
+  if (packet == NULL || packet->getPayloadType() != PAYLOAD_TYPE_GRP_DATA
+      || group_data_gate == FLOOD_RETRY_PATH_GATE_DISABLED) {
+    return general_gate;
+  }
+  if (general_gate == FLOOD_RETRY_PATH_GATE_DISABLED || group_data_gate < general_gate) {
+    return group_data_gate;
+  }
+  return general_gate;
+}
 uint8_t Mesh::getFloodRetryMaxAttempts(const Packet* packet) const {
   (void)packet;
   return FLOOD_RETRY_MAX_ATTEMPTS_DEFAULT;
@@ -424,6 +437,10 @@ void Mesh::onSendComplete(Packet* packet) {
   armFloodRetryOnSendComplete(packet);
 }
 
+void Mesh::onTracePacketQueuedForSend(Packet* packet) {
+  replaceQueuedTraceRetries(packet);
+}
+
 void Mesh::onSendFail(Packet* packet) {
   clearPendingDirectRetryOnSendFail(packet);
   clearPendingFloodRetryOnSendFail(packet);
@@ -913,6 +930,7 @@ void Mesh::clearDirectRetrySlot(int idx) {
   _direct_retries[idx].retry_delay = 0;
   _direct_retries[idx].retry_attempts_sent = 0;
   memset(_direct_retries[idx].retry_key, 0, sizeof(_direct_retries[idx].retry_key));
+  memset(_direct_retries[idx].trace_replacement_key, 0, sizeof(_direct_retries[idx].trace_replacement_key));
   memset(_direct_retries[idx].next_hop_hash, 0, sizeof(_direct_retries[idx].next_hop_hash));
   _direct_retries[idx].next_hop_hash_len = 0;
   _direct_retries[idx].payload_type = 0;
@@ -926,6 +944,26 @@ void Mesh::clearDirectRetrySlot(int idx) {
   if (rebuild_timeout) rebuildNextDirectRetryTimeout();
 }
 
+void Mesh::retireDirectRetrySlot(int idx) {
+  if (idx < 0 || idx >= MAX_DIRECT_RETRY_SLOTS || !_direct_retries[idx].active) {
+    return;
+  }
+
+  Packet* retry = _direct_retries[idx].queued ? _direct_retries[idx].packet : NULL;
+  if (retry != NULL && retry != getOutboundInFlight()) {
+    for (int j = 0; j < _mgr->getOutboundTotal(); j++) {
+      if (_mgr->getOutboundByIdx(j) != retry) continue;
+      Packet* pending = _mgr->removeOutboundByIdx(j);
+      if (pending != NULL) {
+        _direct_retries[idx].packet = NULL;
+        releasePacket(pending);
+      }
+      break;
+    }
+  }
+  clearDirectRetrySlot(idx);
+}
+
 void Mesh::rebuildNextDirectRetryTimeout() {
   bool found = false;
   uint32_t shortest_delay = 0;
@@ -996,6 +1034,62 @@ void Mesh::calculateDirectRetryKey(const Packet* packet, uint8_t* dest_key) cons
   Utils::sha256(dest_key, MAX_HASH_SIZE, &type, 1, packet->payload, packet->payload_len);
 }
 
+bool Mesh::calculateTraceReplacementKey(const Packet* packet, uint8_t* dest_key) const {
+  if (packet == NULL || dest_key == NULL || !packet->isRouteDirect()
+      || packet->getPayloadType() != PAYLOAD_TYPE_TRACE || packet->payload_len < 9) {
+    return false;
+  }
+
+  uint8_t prefix[3] = {
+    PAYLOAD_TYPE_TRACE,
+    (uint8_t)(packet->path_len & 0xFF),
+    (uint8_t)(packet->path_len >> 8)
+  };
+  // Ignore tag/auth (payload bytes 0..7), which change for a new request.
+  // Keep flags, route, and current progress so an older trace that has already
+  // advanced is not mistaken for the stale retry being replaced.
+  Utils::sha256(dest_key, MAX_HASH_SIZE, prefix, sizeof(prefix),
+                &packet->payload[8], packet->payload_len - 8);
+  return true;
+}
+
+void Mesh::replaceQueuedTraceRetries(const Packet* packet) {
+  uint8_t replacement_key[MAX_HASH_SIZE];
+  if (!calculateTraceReplacementKey(packet, replacement_key)) return;
+
+  int replacement_slot = -1;
+  bool found_prior = false;
+  for (int i = 0; i < MAX_DIRECT_RETRY_SLOTS; i++) {
+    if (!_direct_retries[i].active || _direct_retries[i].payload_type != PAYLOAD_TYPE_TRACE
+        || memcmp(replacement_key, _direct_retries[i].trace_replacement_key, MAX_HASH_SIZE) != 0) {
+      continue;
+    }
+    if (_direct_retries[i].trigger_packet == packet) {
+      replacement_slot = i;
+    } else {
+      found_prior = true;
+    }
+  }
+
+  if (!found_prior) return;
+
+  for (int i = 0; i < MAX_DIRECT_RETRY_SLOTS; i++) {
+    if (i == replacement_slot || !_direct_retries[i].active
+        || _direct_retries[i].payload_type != PAYLOAD_TYPE_TRACE
+        || memcmp(replacement_key, _direct_retries[i].trace_replacement_key, MAX_HASH_SIZE) != 0) {
+      continue;
+    }
+    retireDirectRetrySlot(i);
+  }
+
+  // An exact duplicate retry key, or a full retry table, can prevent the new
+  // packet from reserving its slot before it is queued. The prior slots are now
+  // gone, so register the successfully queued packet as the retry owner.
+  if (replacement_slot < 0) {
+    maybeScheduleDirectRetry(packet, getTraceDirectPriority(packet));
+  }
+}
+
 bool Mesh::cancelDirectRetryOnEcho(const Packet* packet) {
   if (_active_direct_retry_count == 0) return false;
 
@@ -1351,6 +1445,8 @@ void Mesh::maybeScheduleDirectRetry(const Packet* packet, uint8_t priority, bool
 
   uint8_t retry_key[MAX_HASH_SIZE];
   calculateDirectRetryKey(packet, retry_key);
+  uint8_t trace_replacement_key[MAX_HASH_SIZE] = { 0 };
+  bool has_trace_replacement_key = calculateTraceReplacementKey(packet, trace_replacement_key);
   for (int i = 0; i < MAX_DIRECT_RETRY_SLOTS; i++) {
     if (_direct_retries[i].active
         && memcmp(retry_key, _direct_retries[i].retry_key, MAX_HASH_SIZE) == 0) {
@@ -1366,6 +1462,17 @@ void Mesh::maybeScheduleDirectRetry(const Packet* packet, uint8_t priority, bool
     }
   }
   if (slot_idx < 0) {
+    if (has_trace_replacement_key) {
+      for (int i = 0; i < MAX_DIRECT_RETRY_SLOTS; i++) {
+        if (_direct_retries[i].active && _direct_retries[i].payload_type == PAYLOAD_TYPE_TRACE
+            && memcmp(trace_replacement_key, _direct_retries[i].trace_replacement_key,
+                      MAX_HASH_SIZE) == 0) {
+          // The post-queue hook will retire this older matching TRACE and use
+          // the slot for the successfully queued replacement.
+          return;
+        }
+      }
+    }
     onDirectRetryEvent("dropped_no_slot", packet, 0, 0, next_hop_hash, next_hop_hash_len);
     onDirectRetryEvent("failure", packet, 0, 0, next_hop_hash, next_hop_hash_len);
     return;
@@ -1374,6 +1481,8 @@ void Mesh::maybeScheduleDirectRetry(const Packet* packet, uint8_t priority, bool
   // Only store retry metadata here; allocate the retry packet after the initial TX really completes.
   uint32_t retry_delay = getDirectRetryAttemptDelay(packet, 0);
   memcpy(_direct_retries[slot_idx].retry_key, retry_key, sizeof(retry_key));
+  memcpy(_direct_retries[slot_idx].trace_replacement_key, trace_replacement_key,
+         sizeof(trace_replacement_key));
   _direct_retries[slot_idx].packet = NULL;
   _direct_retries[slot_idx].trigger_packet = const_cast<Packet*>(packet);
   _direct_retries[slot_idx].retry_started_at = 0;
@@ -1448,20 +1557,7 @@ void Mesh::cancelAllDirectRetries() {
 
   for (int i = 0; i < MAX_DIRECT_RETRY_SLOTS; i++) {
     if (!_direct_retries[i].active) continue;
-
-    Packet* retry = _direct_retries[i].queued ? _direct_retries[i].packet : NULL;
-    if (retry != NULL && retry != getOutboundInFlight()) {
-      for (int j = 0; j < _mgr->getOutboundTotal(); j++) {
-        if (_mgr->getOutboundByIdx(j) != retry) continue;
-        Packet* pending = _mgr->removeOutboundByIdx(j);
-        if (pending != NULL) {
-          _direct_retries[i].packet = NULL;
-          releasePacket(pending);
-        }
-        break;
-      }
-    }
-    clearDirectRetrySlot(i);
+    retireDirectRetrySlot(i);
   }
 }
 
@@ -1501,20 +1597,7 @@ bool Mesh::cancelActiveRetries(const uint8_t retry_key[MAX_HASH_SIZE]) {
       continue;
     }
 
-    Packet* retry = _direct_retries[i].queued ? _direct_retries[i].packet : NULL;
-    if (retry != NULL && retry != getOutboundInFlight()) {
-      for (int j = 0; j < _mgr->getOutboundTotal(); j++) {
-        if (_mgr->getOutboundByIdx(j) == retry) {
-          Packet* pending = _mgr->removeOutboundByIdx(j);
-          if (pending != NULL) {
-            _direct_retries[i].packet = NULL;
-            releasePacket(pending);
-          }
-          break;
-        }
-      }
-    }
-    clearDirectRetrySlot(i);
+    retireDirectRetrySlot(i);
     cancelled = true;
   }
 

src/Mesh.h

@@ -60,6 +60,7 @@ class Mesh : public Dispatcher {
     uint32_t retry_delay;
     uint8_t retry_attempts_sent;
     uint8_t retry_key[MAX_HASH_SIZE];
+    uint8_t trace_replacement_key[MAX_HASH_SIZE];
     uint8_t next_hop_hash[MAX_HASH_SIZE];
     uint8_t next_hop_hash_len;
     uint8_t payload_type;
@@ -104,7 +105,10 @@ class Mesh : public Dispatcher {
   void rebuildNextDirectRetryTimeout();
   void rebuildNextFloodRetryTimeout();
   void clearDirectRetrySlot(int idx);
+  void retireDirectRetrySlot(int idx);
   void calculateDirectRetryKey(const Packet* packet, uint8_t* dest_key) const;
+  bool calculateTraceReplacementKey(const Packet* packet, uint8_t* dest_key) const;
+  void replaceQueuedTraceRetries(const Packet* packet);
   bool cancelDirectRetryOnEcho(const Packet* packet);
   void armDirectRetryOnSendComplete(const Packet* packet);
   void clearPendingDirectRetryOnSendFail(const Packet* packet);
@@ -122,6 +126,7 @@ class Mesh : public Dispatcher {
 
 protected:
   DispatcherAction onRecvPacket(Packet* pkt) override;
+  void onTracePacketQueuedForSend(Packet* packet) override;
   void onSendComplete(Packet* packet) override;
   void onSendFail(Packet* packet) override;
   bool allowPacketTransmit(const Packet* packet) const override;
@@ -236,6 +241,13 @@ protected:
    */
   virtual uint8_t getFloodRetryMaxPathLength(const Packet* packet) const;
 
+  /**
+   * \returns  the stricter of the general flood retry gate and the group-data-specific gate.
+   */
+  static uint8_t applyGroupDataFloodRetryPathGate(const Packet* packet,
+                                                  uint8_t general_gate,
+                                                  uint8_t group_data_gate);
+
   /**
    * \returns  maximum number of FLOOD retry transmissions after the initial TX.
    */

src/helpers/CommonCLI.cpp

@@ -169,6 +169,7 @@ static bool isBasicRetryConfig(const char* config) {
       || configKeyMatches(config, "direct.retry")
       || configKeyMatches(config, "flood.retry.count")
       || configKeyMatches(config, "flood.retry.path")
+      || configKeyMatches(config, "flood.retry.group.path")
       || configKeyMatches(config, "flood.retry.advert");
 }
 
@@ -430,6 +431,9 @@ static void applyFloodRetryPreset(NodePrefs* prefs, uint8_t preset) {
     prefs->flood_retry_attempts = FLOOD_RETRY_ROOFTOP_COUNT;
     prefs->flood_retry_max_path = FLOOD_RETRY_ROOFTOP_MAX_PATH;
   }
+  prefs->flood_retry_group_max_path = prefs->flood_retry_max_path == 0
+      ? FLOOD_RETRY_PATH_GATE_DISABLED
+      : FLOOD_RETRY_GROUP_MAX_PATH_DEFAULT;
 }
 
 static bool parseFloodRetryPathGate(const char* value, uint8_t& path_gate) {
@@ -936,6 +940,7 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) {
     _prefs->ota_max_hops = 3;
 #endif
     _prefs->telemetry_access = TELEMETRY_ACCESS_ALL;
+    _prefs->flood_retry_group_max_path = FLOOD_RETRY_GROUP_MAX_PATH_DEFAULT;
     // A remainder larger than the smallest legacy MQTT gap (864) means an old fork
     // file with the zero-filled gap; detect and recover it below. Anything smaller
     // (upstream/flex 5-byte tails, or the ~384-byte keymind retry tail) takes the
@@ -1183,6 +1188,10 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) {
       file.read((uint8_t *)_prefs->battery_alert_region, sizeof(_prefs->battery_alert_region));
       _prefs->battery_alert_region[sizeof(_prefs->battery_alert_region) - 1] = '\0';
     }
+    if (file.available() >= (int)sizeof(_prefs->flood_retry_group_max_path)) {
+      file.read((uint8_t *)&_prefs->flood_retry_group_max_path,
+                sizeof(_prefs->flood_retry_group_max_path));
+    }
     }
 
     // sanitise bad pref values
@@ -1239,6 +1248,12 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) {
     if (_prefs->flood_retry_max_path != FLOOD_RETRY_PATH_GATE_DISABLED) {
       _prefs->flood_retry_max_path = constrain(_prefs->flood_retry_max_path, 0, 63);
     }
+    if (_prefs->flood_retry_group_max_path != FLOOD_RETRY_PATH_GATE_DISABLED) {
+      _prefs->flood_retry_group_max_path = constrain(_prefs->flood_retry_group_max_path, 0, 63);
+    }
+    if (_prefs->flood_retry_max_path == 0) {
+      _prefs->flood_retry_group_max_path = FLOOD_RETRY_PATH_GATE_DISABLED;
+    }
     _prefs->flood_retry_bridge_enabled = constrain(_prefs->flood_retry_bridge_enabled, 0, 1);
     _prefs->flood_retry_advert_enabled = constrain(_prefs->flood_retry_advert_enabled, 0, 1);
     _prefs->battery_alert_enabled = constrain(_prefs->battery_alert_enabled, 0, 1);
@@ -1402,7 +1417,9 @@ void CommonCLI::savePrefs(FILESYSTEM* fs) {
     file.write((uint8_t *)&_prefs->rx_ps_level, sizeof(_prefs->rx_ps_level));                       // 820
     file.write((uint8_t *)&_prefs->rx_ps_preamble, sizeof(_prefs->rx_ps_preamble));                 // 821
     file.write((uint8_t *)_prefs->battery_alert_region, sizeof(_prefs->battery_alert_region));      // 822
-    // next: 853
+    file.write((uint8_t *)&_prefs->flood_retry_group_max_path,
+               sizeof(_prefs->flood_retry_group_max_path));                                        // 853
+    // next: 854
 
     file.close();
   }
@@ -2612,6 +2629,21 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep
     uint8_t path_gate;
     if (parseFloodRetryPathGate(&config[17], path_gate)) {
       _prefs->flood_retry_max_path = path_gate;
+      if (path_gate == 0) {
+        _prefs->flood_retry_group_max_path = FLOOD_RETRY_PATH_GATE_DISABLED;
+      }
+      _prefs->retry_preset = RETRY_PRESET_CUSTOM;
+      savePrefs();
+      strcpy(reply, "OK");
+    } else {
+      strcpy(reply, "Error, must be 0-63 or off");
+    }
+  } else if (memcmp(config, "flood.retry.group.path ", 23) == 0) {
+    uint8_t path_gate;
+    if (parseFloodRetryPathGate(&config[23], path_gate)) {
+      _prefs->flood_retry_group_max_path = _prefs->flood_retry_max_path == 0
+          ? FLOOD_RETRY_PATH_GATE_DISABLED
+          : path_gate;
       _prefs->retry_preset = RETRY_PRESET_CUSTOM;
       savePrefs();
       strcpy(reply, "OK");
@@ -3029,6 +3061,10 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep
     char path_gate[8];
     formatFloodRetryPathGate(path_gate, _prefs->flood_retry_max_path);
     sprintf(reply, "> %s", path_gate);
+  } else if (memcmp(config, "flood.retry.group.path", 22) == 0) {
+    char path_gate[8];
+    formatFloodRetryPathGate(path_gate, _prefs->flood_retry_group_max_path);
+    sprintf(reply, "> %s", path_gate);
   } else if (memcmp(config, "flood.retry.prefixes", 20) == 0) {
     formatFloodRetryPrefixList(tmp, _prefs->flood_retry_prefixes, FLOOD_RETRY_PREFIX_SLOTS);
     sprintf(reply, "> %s", tmp[0] ? tmp : "none");

src/helpers/CommonCLI.h

@@ -50,6 +50,7 @@
 #define FLOOD_RETRY_MOBILE_COUNT         15
 #define FLOOD_RETRY_MOBILE_MAX_PATH       1
 #define FLOOD_RETRY_ADVERT_DEFAULT        0
+#define FLOOD_RETRY_GROUP_MAX_PATH_DEFAULT 1
 
 #define BATTERY_ALERT_LOW_PERCENT_DEFAULT       20
 #define BATTERY_ALERT_CRITICAL_PERCENT_DEFAULT  10
@@ -215,6 +216,7 @@ struct NodePrefs { // persisted to file
   uint8_t rx_ps_level;      // 0 = manual/explicit us timings; 1..10 = level-derived (auto-retunes on SF/BW change)
   uint8_t rx_ps_preamble;   // 0 = auto (derive from SF); else 16 or 32 = explicit override for level calc
   char battery_alert_region[31]; // named scope for low-battery floods; empty = no alert scope
+  uint8_t flood_retry_group_max_path; // PAYLOAD_TYPE_GRP_DATA retry path gate; 0xFF = use only the general gate
 };
 
 #ifdef WITH_MQTT_BRIDGE