dandri/HaloKeymind
1
0
Fork 0
mirror of https://github.com/mikecarper/MeshCore.git synced 2026-10-20 15:01:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce neighbor discovery RAM without changing published results

Browse Source
This commit is contained in:
mikecarper
2026-10-10 01:30:09 -07:00
parent 894d033503
commit 04f98ea44c
5 changed files with 261 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/simple_repeater/MyMesh.cpp
+5
View File
@@ -13428,6 +13428,11 @@ void MyMesh::resetNeighborDiscoverJsonBudget() {
// progress metadata; UINT32_MAX likewise reserves the widest heard-age value.
// If this result cannot fit, stop before transmitting another scope request.
bool MyMesh::completeNeighborDiscoverEntry() {
if ((size_t)neighbor_discover_next >= NEIGHBOR_SCOPE_RESULTS) {
neighbor_discover_truncated = true;
finishNeighborDiscover();
return false;
}
NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next];
char pubkey_hex[65];
mesh::Utils::toHex(pubkey_hex, entry.id.pub_key, PUB_KEY_SIZE);
examples/simple_repeater/MyMesh.h
+4 -4
View File
@@ -701,9 +701,9 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks
struct NeighborDiscoverEntry {
mesh::Identity id; // immutable snapshot: neighbour table can change mid-pass
uint32_t heard_timestamp;
int8_t snr; // multiplied by 4
int16_t rssi; // dBm from the last packet heard from this neighbour
uint32_t tag; // anon-regions request tag we're waiting on
int16_t rssi; // dBm from the last packet heard from this neighbour
int8_t snr; // multiplied by 4
uint8_t status; // NeighborDiscoverStatus
};
NeighborDiscoverEntry neighbor_discover[MAX_NEIGHBOURS];
@@ -711,8 +711,8 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks
// receives scope queries. Keep every neighbor snapshot, without reserving
// 96 response bytes for each entry that will never be queried.
static constexpr size_t NEIGHBOR_SCOPE_RESULTS =
MAX_NEIGHBOURS < MQTTBridge::NEIGHBORS_MAX_PUBLISH_ENTRIES + 1
? MAX_NEIGHBOURS : MQTTBridge::NEIGHBORS_MAX_PUBLISH_ENTRIES + 1;
MAX_NEIGHBOURS < MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY
? MAX_NEIGHBOURS : MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY;
char neighbor_discover_scopes[NEIGHBOR_SCOPE_RESULTS][96];
uint8_t neighbor_discover_count;
uint8_t neighbor_discover_next; // newest-first entry currently being queried
src/helpers/bridges/MQTTBridge.h
+12
View File
@@ -677,6 +677,18 @@ public:
static const int NEIGHBORS_MAX_PUBLISH_ENTRIES = 20;
#endif
// Discovery budgets each row with a 64-character public key, UINT32_MAX
// heard age and a status of at least "timeout". Its shortest measured JSON
// is 144 bytes. Reserve scope results only for rows that could fit the
// existing JSON buffer, plus one overflow sentinel. Serializer tests keep
// this lower bound tied to completeNeighborDiscoverEntry's measurement.
// The neighbor cache and actual per-publish entry limit remain unchanged.
static const size_t NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES = 144;
static const size_t NEIGHBORS_SCOPE_RESULT_CAPACITY =
(NEIGHBORS_MAX_PUBLISH_ENTRIES < NEIGHBORS_JSON_BUFFER_SIZE / NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES
? NEIGHBORS_MAX_PUBLISH_ENTRIES
: NEIGHBORS_JSON_BUFFER_SIZE / NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES) + 1;
// Called by the mesh (Core 1) once a neighbor-discovery pass has built the
// table JSON. Copies it into the persistent PSRAM buffer and raises the
// publish-pending flag for the MQTT task; a request is dropped if one is
test/fixtures/neighbor_scope_storage.cpp
Vendored
+147
View File
@@ -0,0 +1,147 @@
#define BOARD_HAS_PSRAM 1
#define MAX_NEIGHBOURS 254
#define PUB_KEY_SIZE 32
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <initializer_list>
#include <string>
#include <vector>
#include <helpers/MQTTPayloadBuilder.h>
namespace mesh {
struct Identity { uint8_t pub_key[32]; };
struct Utils {
static void toHex(char* out, const uint8_t* key, size_t size) {
for(size_t i=0;i<size;++i)snprintf(out+2*i,3,"%02x",key[i]);
}
};
}
struct MQTTBridge {
@BRIDGE_CONSTANTS@
};
struct MQTTMessageBuilder : MQTTPayloadBuilder {
static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& entry) {
size_t measured=MQTTPayloadBuilder::measureNeighborsMessageEntry(entry);
#ifndef TEST_PUBLICATION_NEGATIVE_CONTROL
// Production discovery calls this measurement before advancing its cursor.
// Check the bound against its actual fields as well as formatter cases.
assert(measured>=MQTTBridge::NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES);
#endif
return measured;
}
};
struct Clock { uint32_t getCurrentTime() { return UINT32_MAX; } };
struct MyMesh {
enum { ND_UNSENT, ND_QUEUED, ND_PENDING, ND_RESPONDED, ND_TIMEOUT, ND_SEND_FAILED };
@LAYOUT@
uint8_t neighbor_discover_next=0, neighbor_discover_count=254;
uint8_t neighbor_discover_publish_count=0;
size_t neighbor_discover_json_size=0;
bool neighbor_discover_truncated=false, finished=false;
Clock clock;
Clock* getRTCClock() { return &clock; }
void touchNeighbourHeard(const mesh::Identity&, uint32_t, float, int16_t) {}
void finishNeighborDiscover() { finished=true; }
bool completeNeighborDiscoverEntry();
bool handleNeighborDiscoverResponse(int,const uint8_t*,size_t,float,int16_t);
};
struct LegacyMesh {
enum { ND_UNSENT, ND_QUEUED, ND_PENDING, ND_RESPONDED, ND_TIMEOUT, ND_SEND_FAILED };
@LEGACY_LAYOUT@
uint8_t neighbor_discover_next=0, neighbor_discover_count=254;
uint8_t neighbor_discover_publish_count=0;
size_t neighbor_discover_json_size=0;
bool neighbor_discover_truncated=false, finished=false;
Clock clock;
Clock* getRTCClock() { return &clock; }
void touchNeighbourHeard(const mesh::Identity&, uint32_t, float, int16_t) {}
void finishNeighborDiscover() { finished=true; }
bool completeNeighborDiscoverEntry();
bool handleNeighborDiscoverResponse(int,const uint8_t*,size_t,float,int16_t);
};
@METHODS@
@LEGACY_METHODS@
template<class Mesh> std::string collect(Mesh& mesh,size_t scope_bytes,int state,int rssi,float snr) {
const char* timestamp="2026-10-10T01:23:45Z";
const std::string own_key(64,'a');
mesh.neighbor_discover_json_size=MQTTPayloadBuilder::measureNeighborsMessageBase(
"Seattle",own_key.c_str(),timestamp,"sea;pdx","sea",254);
std::vector<std::string> keys; keys.reserve(254);
std::vector<MQTTPayloadBuilder::NeighborsMessageEntry> entries;
for(int i=0;i<254;++i) {
auto& row=mesh.neighbor_discover[i];
memset(row.id.pub_key,uint8_t(i),sizeof(row.id.pub_key));
row.status=uint8_t(state); row.snr=int8_t(snr*4); row.rssi=int16_t(rssi);
if(state==Mesh::ND_RESPONDED) {
row.status=Mesh::ND_PENDING; row.tag=123;
std::vector<uint8_t> response(8+scope_bytes,'x');
memcpy(response.data(),&row.tag,4);
assert(mesh.handleNeighborDiscoverResponse(i,response.data(),response.size(),snr,int16_t(rssi)));
}
if(!mesh.completeNeighborDiscoverEntry())break;
char key[65]; mesh::Utils::toHex(key,row.id.pub_key,32); keys.emplace_back(key);
entries.push_back({keys.back().c_str(),row.snr/4.0f,UINT32_MAX,
mesh.neighbor_discover_scopes[i],state==Mesh::ND_RESPONDED ? "responded"
: (state==Mesh::ND_SEND_FAILED ? "send_failed" : "timeout"),rssi,false});
}
assert(mesh.finished && mesh.neighbor_discover_truncated);
assert(entries.size()==mesh.neighbor_discover_publish_count);
JsonDocument document;
std::vector<char> json(MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE);
int length=MQTTPayloadBuilder::buildNeighborsMessage(document,"Seattle",own_key.c_str(),timestamp,
"sea;pdx","sea",entries.data(),int(entries.size()),json.data(),json.size(),254,
int(entries.size())+1,true);
assert(length>0 && size_t(length)<json.size());
assert(document["total_neighbors"].as<int>()==254 && document["truncated"].as<bool>());
assert(document["neighbors"].as<JsonArray>().size()==entries.size());
return std::string(json.data(),size_t(length));
}
int main() {
MyMesh memory={}; LegacyMesh legacy_memory={};
static_assert(sizeof(memory.neighbor_discover)/sizeof(memory.neighbor_discover[0])==254,
"keep the complete neighbor snapshot capacity");
#ifndef TEST_PUBLICATION_NEGATIVE_CONTROL
assert(sizeof(MyMesh::NeighborDiscoverEntry)==44);
assert(sizeof(LegacyMesh::NeighborDiscoverEntry)==48);
assert(sizeof(memory.neighbor_discover_scopes)==72*96);
assert(sizeof(legacy_memory.neighbor_discover_scopes)-sizeof(memory.neighbor_discover_scopes)==17472);
assert(sizeof(legacy_memory.neighbor_discover)-sizeof(memory.neighbor_discover)==1016);
#else
(void)legacy_memory;
#endif
const std::string key(64,'0');
MQTTPayloadBuilder::NeighborsMessageEntry minimum={key.c_str(),0,0,"","",0,false};
assert(MQTTPayloadBuilder::measureNeighborsMessageEntry(minimum)==128);
#ifndef TEST_PUBLICATION_NEGATIVE_CONTROL
minimum.heard_secs_ago=UINT32_MAX; minimum.status="timeout";
assert(MQTTPayloadBuilder::measureNeighborsMessageEntry(minimum)==144);
assert(MQTTBridge::NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES<=144);
for(int rssi : {0,-1,-100,INT16_MIN})
for(float snr : {0.0f,-32.0f,12.75f})
for(const char* status : {"responded","timeout","send_failed"}) {
auto entry=minimum; entry.rssi=rssi;
entry.snr=snr; entry.status=status;
assert(MQTTPayloadBuilder::measureNeighborsMessageEntry(entry)>=MQTTBridge::NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES);
}
#endif
for(size_t scope : {size_t(0),size_t(1),size_t(31),size_t(95),size_t(200)})
for(int state : {MyMesh::ND_RESPONDED,MyMesh::ND_TIMEOUT,MyMesh::ND_SEND_FAILED})
for(int rssi : {0,-100})
for(float snr : {0.0f,12.75f}) {
MyMesh current={}; LegacyMesh baseline={};
auto actual=collect(current,scope,state,rssi,snr);
auto old=collect(baseline,scope,state,rssi,snr);
assert(actual==old && current.neighbor_discover_publish_count==baseline.neighbor_discover_publish_count);
assert(current.neighbor_discover_next<MyMesh::NEIGHBOR_SCOPE_RESULTS);
}
// Explicit bounds reject an unexpected late response and safely finish an
// invalid cursor, even if future serializer changes violate the size proof.
memory.neighbor_discover_next=MyMesh::NEIGHBOR_SCOPE_RESULTS;
uint8_t response[8]={};
assert(!memory.handleNeighborDiscoverResponse(int(MyMesh::NEIGHBOR_SCOPE_RESULTS),response,8,0,0));
assert(!memory.completeNeighborDiscoverEntry() && memory.finished && memory.neighbor_discover_truncated);
puts("real neighbor serialization and complete publication checks passed");
}
test/test_neighbor_scope_storage.py
+93 -4
View File
@@ -1,6 +1,8 @@
"""Exercise production scope storage and truncation with 254 neighbors."""
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import unittest
@@ -10,6 +12,11 @@ ROOT = Path(__file__).resolve().parents[1]
class NeighborScopeStorageTest(unittest.TestCase):
def bridge_constants(self):
source = (ROOT / "src/helpers/bridges/MQTTBridge.h").read_text()
start = source.index(" #if defined(BOARD_HAS_PSRAM)", source.index("// Single source of truth for the neighbors JSON size"))
return source[start:source.index(" // Called by the mesh", start)]
def test_response_bounds_and_publish_prefix(self):
header = (ROOT / "examples/simple_repeater/MyMesh.h").read_text()
source = (ROOT / "examples/simple_repeater/MyMesh.cpp").read_text()
@@ -30,8 +37,7 @@ struct Identity { uint8_t pub_key[32]; };
struct Utils { static void toHex(char* out, const uint8_t*, size_t) { out[0] = 0; } };
}
struct MQTTBridge {
static constexpr int NEIGHBORS_MAX_PUBLISH_ENTRIES = 20;
static constexpr size_t NEIGHBORS_JSON_BUFFER_SIZE = 8192;
''' + self.bridge_constants() + r'''
};
struct MQTTMessageBuilder {
struct NeighborsMessageEntry {
@@ -44,8 +50,8 @@ struct Clock { uint32_t getCurrentTime() { return 100; } };
struct MyMesh {
enum { ND_UNSENT, ND_QUEUED, ND_PENDING, ND_RESPONDED, ND_TIMEOUT, ND_SEND_FAILED };
''' + layout + r'''
unsigned neighbor_discover_next = 0, neighbor_discover_count = 254;
unsigned neighbor_discover_publish_count = 0;
uint8_t neighbor_discover_next = 0, neighbor_discover_count = 254;
uint8_t neighbor_discover_publish_count = 0;
size_t neighbor_discover_json_size = 0;
bool neighbor_discover_truncated = false, finished = false;
Clock clock;
@@ -86,6 +92,89 @@ int main() {
str(path / "test.cpp"), "-o", str(path / "test")], check=True)
subprocess.run([str(path / "test")], check=True)
def real_serializer_fixture(self):
header = (ROOT / "examples/simple_repeater/MyMesh.h").read_text()
source = (ROOT / "examples/simple_repeater/MyMesh.cpp").read_text()
layout = header[header.index(" struct NeighborDiscoverEntry {"):
header.index(" uint8_t neighbor_discover_count;")]
methods = "\n".join(extract_braced(source, signature) for signature in (
"bool MyMesh::completeNeighborDiscoverEntry()",
"bool MyMesh::handleNeighborDiscoverResponse("))
legacy_layout = layout.replace(
" static constexpr size_t NEIGHBOR_SCOPE_RESULTS =\n"
" MAX_NEIGHBOURS < MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY\n"
" ? MAX_NEIGHBOURS : MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY;",
" static constexpr size_t NEIGHBOR_SCOPE_RESULTS = MAX_NEIGHBOURS;")
legacy_layout = legacy_layout.replace(
" uint32_t tag; // anon-regions request tag we're waiting on\n"
" int16_t rssi; // dBm from the last packet heard from this neighbour\n"
" int8_t snr; // multiplied by 4\n",
" int8_t snr; // multiplied by 4\n"
" int16_t rssi; // dBm from the last packet heard from this neighbour\n"
" uint32_t tag; // anon-regions request tag we're waiting on\n")
self.assertNotEqual(layout, legacy_layout)
fixture = (ROOT / "test/fixtures/neighbor_scope_storage.cpp").read_text()
return (fixture.replace("@BRIDGE_CONSTANTS@", self.bridge_constants())
.replace("@LAYOUT@", layout).replace("@METHODS@", methods)
.replace("@LEGACY_LAYOUT@", legacy_layout)
.replace("@LEGACY_METHODS@", methods.replace("MyMesh::", "LegacyMesh::")))
def compile_real_serializer(self, code, expect_rejection=False):
compiler = shutil.which("g++") or shutil.which("clang++")
self.assertIsNotNone(compiler)
json_header = next((ROOT / ".pio/libdeps").glob("*/ArduinoJson/src/ArduinoJson.h"), None)
self.assertIsNotNone(json_header, "actual native ArduinoJson headers are required")
with tempfile.TemporaryDirectory(prefix="meshcore-neighbor-scope-") as temporary:
work = Path(temporary)
generated = work / "scope.cpp"
generated.write_text(code)
binary = work / "scope"
sanitizers = (["-fsanitize=address,undefined", "-fno-sanitize-recover=all",
"-fno-pie", "-no-pie"] if sys.platform.startswith("linux") else [])
compiled = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror",
"-Wno-missing-field-initializers", *sanitizers,
"-I" + str(json_header.parent), "-I" + str(ROOT / "src"),
"-isystem", str(ROOT / "test/mocks"),
str(generated), str(ROOT / "src/helpers/MQTTPayloadBuilder.cpp"),
str(ROOT / "src/helpers/TxtDataHelpers.cpp"),
"-o", str(binary)], capture_output=True, text=True, timeout=60)
self.assertEqual(compiled.returncode, 0, compiled.stderr)
checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30)
if expect_rejection:
self.assertNotEqual(checked.returncode, 0, checked.stdout + checked.stderr)
self.assertIn("Assertion", checked.stderr)
else:
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
self.assertIn("real neighbor serialization", checked.stdout)
def test_full_254_neighbors_keep_every_publishable_entry_and_json(self):
self.compile_real_serializer(self.real_serializer_fixture())
def test_reintroduced_full_scope_array_fails_runtime_memory_regression(self):
fixture = self.real_serializer_fixture()
old = fixture
start = old.index("struct MyMesh")
end = old.index("struct LegacyMesh")
candidate = old[start:end]
candidate = candidate.replace(
" static constexpr size_t NEIGHBOR_SCOPE_RESULTS =\n"
" MAX_NEIGHBOURS < MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY\n"
" ? MAX_NEIGHBOURS : MQTTBridge::NEIGHBORS_SCOPE_RESULT_CAPACITY;",
" static constexpr size_t NEIGHBOR_SCOPE_RESULTS = MAX_NEIGHBOURS;")
old = old[:start] + candidate + old[end:]
self.assertNotEqual(old, fixture)
self.compile_real_serializer(old, expect_rejection=True)
def test_over_restrictive_bound_fails_actual_publication_regression(self):
fixture = self.real_serializer_fixture()
old = fixture.replace("NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES = 144;",
"NEIGHBORS_MIN_DISCOVERY_ENTRY_JSON_BYTES = 512;", 1)
self.assertNotEqual(old, fixture)
# Run through publication comparison instead of stopping at the memory
# or bound assertion: a needlessly small array loses actual JSON rows.
old = "#define TEST_PUBLICATION_NEGATIVE_CONTROL 1\n" + old
self.compile_real_serializer(old, expect_rejection=True)
if __name__ == "__main__":
unittest.main()