From 04f98ea44ceec53aaa1f06ba6926572c91c69b4a Mon Sep 17 00:00:00 2001 From: mikecarper Date: Sat, 10 Oct 2026 01:30:09 -0700 Subject: [PATCH] Reduce neighbor discovery RAM without changing published results --- examples/simple_repeater/MyMesh.cpp | 5 + examples/simple_repeater/MyMesh.h | 8 +- src/helpers/bridges/MQTTBridge.h | 12 ++ test/fixtures/neighbor_scope_storage.cpp | 147 +++++++++++++++++++++++ test/test_neighbor_scope_storage.py | 97 ++++++++++++++- 5 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 test/fixtures/neighbor_scope_storage.cpp diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index d2ee7095a..a346e6348 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -13428,6 +13428,11 @@ void MyMesh::resetNeighborDiscoverJsonBudget() { // progress metadata; UINT32_MAX likewise reserves the widest heard-age value. // If this result cannot fit, stop before transmitting another scope request. bool MyMesh::completeNeighborDiscoverEntry() { + if ((size_t)neighbor_discover_next >= NEIGHBOR_SCOPE_RESULTS) { + neighbor_discover_truncated = true; + finishNeighborDiscover(); + return false; + } NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; char pubkey_hex[65]; mesh::Utils::toHex(pubkey_hex, entry.id.pub_key, PUB_KEY_SIZE); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 78d249f8c..050d48058 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -701,9 +701,9 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks struct NeighborDiscoverEntry { mesh::Identity id; // immutable snapshot: neighbour table can change mid-pass uint32_t heard_timestamp; - int8_t snr; // multiplied by 4 - int16_t rssi; // dBm from the last packet heard from this neighbour uint32_t tag; // anon-regions request tag we're waiting on + int16_t rssi; // dBm from the last packet heard from this neighbour + int8_t snr; // multiplied by 4 uint8_t status; // NeighborDiscoverStatus }; NeighborDiscoverEntry neighbor_discover[MAX_NEIGHBOURS]; @@ -711,8 +711,8 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks // receives scope queries. Keep every neighbor snapshot, without reserving // 96 response bytes for each entry that will never be queried. static constexpr size_t NEIGHBOR_SCOPE_RESULTS = - MAX_NEIGHBOURS < MQTTBridge::NEIGHBORS_MAX_PUBLISH_ENTRIES + 1 - ? MAX_NEIGHBOURS : MQTTBridge::NEIGHBORS_MAX_PUBLISH_ENTRIES + 1; + MAX_NEIGHBOURS < MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY + ? MAX_NEIGHBOURS : MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY; char neighbor_discover_scopes[NEIGHBOR_SCOPE_RESULTS][96]; uint8_t neighbor_discover_count; uint8_t neighbor_discover_next; // newest-first entry currently being queried diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 0ff5a9d27..115988eaa 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -677,6 +677,18 @@ public: static const int NEIGHBORS_MAX_PUBLISH_ENTRIES = 20; #endif + // Discovery budgets each row with a 64-character public key, UINT32_MAX + // heard age and a status of at least "timeout". Its shortest measured JSON + // is 144 bytes. Reserve scope results only for rows that could fit the + // existing JSON buffer, plus one overflow sentinel. Serializer tests keep + // this lower bound tied to completeNeighborDiscoverEntry's measurement. + // The neighbor cache and actual per-publish entry limit remain unchanged. + static const size_t NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES = 144; + static const size_t NEIGHBORS_SCOPE_RESULT_CAPACITY = + (NEIGHBORS_MAX_PUBLISH_ENTRIES < NEIGHBORS_JSON_BUFFER_SIZE / NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES + ? NEIGHBORS_MAX_PUBLISH_ENTRIES + : NEIGHBORS_JSON_BUFFER_SIZE / NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES) + 1; + // Called by the mesh (Core 1) once a neighbor-discovery pass has built the // table JSON. Copies it into the persistent PSRAM buffer and raises the // publish-pending flag for the MQTT task; a request is dropped if one is diff --git a/test/fixtures/neighbor_scope_storage.cpp b/test/fixtures/neighbor_scope_storage.cpp new file mode 100644 index 000000000..1c07e2e3e --- /dev/null +++ b/test/fixtures/neighbor_scope_storage.cpp @@ -0,0 +1,147 @@ +#define BOARD_HAS_PSRAM 1 +#define MAX_NEIGHBOURS 254 +#define PUB_KEY_SIZE 32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace mesh { +struct Identity { uint8_t pub_key[32]; }; +struct Utils { + static void toHex(char* out, const uint8_t* key, size_t size) { + for(size_t i=0;i=MQTTBridge::NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES); +#endif + return measured; + } +}; +struct Clock { uint32_t getCurrentTime() { return UINT32_MAX; } }; +struct MyMesh { + enum { ND_UNSENT, ND_QUEUED, ND_PENDING, ND_RESPONDED, ND_TIMEOUT, ND_SEND_FAILED }; +@LAYOUT@ + uint8_t neighbor_discover_next=0, neighbor_discover_count=254; + uint8_t neighbor_discover_publish_count=0; + size_t neighbor_discover_json_size=0; + bool neighbor_discover_truncated=false, finished=false; + Clock clock; + Clock* getRTCClock() { return &clock; } + void touchNeighbourHeard(const mesh::Identity&, uint32_t, float, int16_t) {} + void finishNeighborDiscover() { finished=true; } + bool completeNeighborDiscoverEntry(); + bool handleNeighborDiscoverResponse(int,const uint8_t*,size_t,float,int16_t); +}; +struct LegacyMesh { + enum { ND_UNSENT, ND_QUEUED, ND_PENDING, ND_RESPONDED, ND_TIMEOUT, ND_SEND_FAILED }; +@LEGACY_LAYOUT@ + uint8_t neighbor_discover_next=0, neighbor_discover_count=254; + uint8_t neighbor_discover_publish_count=0; + size_t neighbor_discover_json_size=0; + bool neighbor_discover_truncated=false, finished=false; + Clock clock; + Clock* getRTCClock() { return &clock; } + void touchNeighbourHeard(const mesh::Identity&, uint32_t, float, int16_t) {} + void finishNeighborDiscover() { finished=true; } + bool completeNeighborDiscoverEntry(); + bool handleNeighborDiscoverResponse(int,const uint8_t*,size_t,float,int16_t); +}; +@METHODS@ +@LEGACY_METHODS@ +template std::string collect(Mesh& mesh,size_t scope_bytes,int state,int rssi,float snr) { + const char* timestamp="2026-10-10T01:23:45Z"; + const std::string own_key(64,'a'); + mesh.neighbor_discover_json_size=MQTTPayloadBuilder::measureNeighborsMessageBase( + "Seattle",own_key.c_str(),timestamp,"sea;pdx","sea",254); + std::vector keys; keys.reserve(254); + std::vector entries; + for(int i=0;i<254;++i) { + auto& row=mesh.neighbor_discover[i]; + memset(row.id.pub_key,uint8_t(i),sizeof(row.id.pub_key)); + row.status=uint8_t(state); row.snr=int8_t(snr*4); row.rssi=int16_t(rssi); + if(state==Mesh::ND_RESPONDED) { + row.status=Mesh::ND_PENDING; row.tag=123; + std::vector response(8+scope_bytes,'x'); + memcpy(response.data(),&row.tag,4); + assert(mesh.handleNeighborDiscoverResponse(i,response.data(),response.size(),snr,int16_t(rssi))); + } + if(!mesh.completeNeighborDiscoverEntry())break; + char key[65]; mesh::Utils::toHex(key,row.id.pub_key,32); keys.emplace_back(key); + entries.push_back({keys.back().c_str(),row.snr/4.0f,UINT32_MAX, + mesh.neighbor_discover_scopes[i],state==Mesh::ND_RESPONDED ? "responded" + : (state==Mesh::ND_SEND_FAILED ? "send_failed" : "timeout"),rssi,false}); + } + assert(mesh.finished && mesh.neighbor_discover_truncated); + assert(entries.size()==mesh.neighbor_discover_publish_count); + JsonDocument document; + std::vector json(MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE); + int length=MQTTPayloadBuilder::buildNeighborsMessage(document,"Seattle",own_key.c_str(),timestamp, + "sea;pdx","sea",entries.data(),int(entries.size()),json.data(),json.size(),254, + int(entries.size())+1,true); + assert(length>0 && size_t(length)()==254 && document["truncated"].as()); + assert(document["neighbors"].as().size()==entries.size()); + return std::string(json.data(),size_t(length)); +} +int main() { + MyMesh memory={}; LegacyMesh legacy_memory={}; + static_assert(sizeof(memory.neighbor_discover)/sizeof(memory.neighbor_discover[0])==254, + "keep the complete neighbor snapshot capacity"); +#ifndef TEST_PUBLICATION_NEGATIVE_CONTROL + assert(sizeof(MyMesh::NeighborDiscoverEntry)==44); + assert(sizeof(LegacyMesh::NeighborDiscoverEntry)==48); + assert(sizeof(memory.neighbor_discover_scopes)==72*96); + assert(sizeof(legacy_memory.neighbor_discover_scopes)-sizeof(memory.neighbor_discover_scopes)==17472); + assert(sizeof(legacy_memory.neighbor_discover)-sizeof(memory.neighbor_discover)==1016); +#else + (void)legacy_memory; +#endif + const std::string key(64,'0'); + MQTTPayloadBuilder::NeighborsMessageEntry minimum={key.c_str(),0,0,"","",0,false}; + assert(MQTTPayloadBuilder::measureNeighborsMessageEntry(minimum)==128); +#ifndef TEST_PUBLICATION_NEGATIVE_CONTROL + minimum.heard_secs_ago=UINT32_MAX; minimum.status="timeout"; + assert(MQTTPayloadBuilder::measureNeighborsMessageEntry(minimum)==144); + assert(MQTTBridge::NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES<=144); + for(int rssi : {0,-1,-100,INT16_MIN}) + for(float snr : {0.0f,-32.0f,12.75f}) + for(const char* status : {"responded","timeout","send_failed"}) { + auto entry=minimum; entry.rssi=rssi; + entry.snr=snr; entry.status=status; + assert(MQTTPayloadBuilder::measureNeighborsMessageEntry(entry)>=MQTTBridge::NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES); + } +#endif + for(size_t scope : {size_t(0),size_t(1),size_t(31),size_t(95),size_t(200)}) + for(int state : {MyMesh::ND_RESPONDED,MyMesh::ND_TIMEOUT,MyMesh::ND_SEND_FAILED}) + for(int rssi : {0,-100}) + for(float snr : {0.0f,12.75f}) { + MyMesh current={}; LegacyMesh baseline={}; + auto actual=collect(current,scope,state,rssi,snr); + auto old=collect(baseline,scope,state,rssi,snr); + assert(actual==old && current.neighbor_discover_publish_count==baseline.neighbor_discover_publish_count); + assert(current.neighbor_discover_next