diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index eb7989195..1337b7f51 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -154,6 +154,7 @@ jobs: python3 -B test/test_common_radio_persistence.py -v python3 -B test/test_common_cli_save_results.py -v python3 -B test/test_common_prefs_commit.py -v + python3 -B test/test_flood_policy_capacity.py -v python3 -B test/test_mqtt_prefs_commit.py -v python3 -B test/test_identity_and_settings_recovery.py -v python3 -B test/test_contact_cache.py -v diff --git a/build_legacy.sh b/build_legacy.sh index 6c6f1fd5e..741413deb 100755 --- a/build_legacy.sh +++ b/build_legacy.sh @@ -3769,6 +3769,16 @@ apply_nrf52_size_profile() { record_build_reduction \ "mesh.flood_rules limited to 4 by measured internal RAM; complete rule engine, 8 KiB loop stack, 50 neighbors, color display, GPS, sensors, and OTA retained" ;; + rak_3401_repeater_unified_lora_ota) + # Internal staging keeps a reset-retained 64 KiB arena even when the + # unified image can also select QSPI. Its complete sensor profile is + # 1,508 bytes short at 63 rules. Reserve 3,200 fewer rule-table bytes; + # keep the 8 KiB stack, 50 neighbors, all sensors and both OTA stores. + append_platformio_build_unflags "-DFLOOD_PACKET_FILTER_SLOTS=63" + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFLOOD_PACKET_FILTER_SLOTS=47" + record_build_reduction \ + "mesh.flood_rules limited to 47 by measured internal RAM; complete rule engine, 8 KiB loop stack, 50 neighbors, GPS, sensors, and unified internal/QSPI OTA retained" + ;; esac } diff --git a/docs/_javascript/firmware_picker.js b/docs/_javascript/firmware_picker.js index 44d5f13ca..49af113d5 100644 --- a/docs/_javascript/firmware_picker.js +++ b/docs/_javascript/firmware_picker.js @@ -254,8 +254,13 @@ const acceptedTags = new Set(RELEASE_PAGE_PREFIXES.map(function (prefix) { return prefix + familyTag; })); + const chunkGroups = ["companion"].concat(RELEASE_PAGE_PREFIXES.filter(Boolean).map(function (prefix) { + return prefix.slice(0, -1); + })); + const chunkTag = new RegExp("^(?:" + chunkGroups.map(escapeRegExp).join("|") + + ")-(?:[2-9]|[1-9][0-9]+)-" + escapeRegExp(familyTag) + "$"); const familyReleases = publicReleases.filter(function (release) { - return acceptedTags.has(release.tag_name); + return acceptedTags.has(release.tag_name) || chunkTag.test(release.tag_name); }).sort(function (a, b) { return releaseDate(b).localeCompare(releaseDate(a)); }); diff --git a/docs/releases/1.17.1.9.md b/docs/releases/1.17.1.9.md index 8ea980773..f6630d224 100644 --- a/docs/releases/1.17.1.9.md +++ b/docs/releases/1.17.1.9.md @@ -34,6 +34,15 @@ layout before installing an image. color framebuffer and OTA arena. Their capability manifests record this capacity limit; heap qualification margins are unchanged. +- The RAK3401 unified repeater reserves 47 flood-rule entries so its full + sensor profile, 8 KiB loop stack, 50 neighbors and internal/QSPI OTA stores + fit the runtime RAM requirement. Full and reduced sensor images use the + same table capacity; the capability reports record this limit. +- Flood-policy upgrades accept inactive trailing rows from larger tables. + If active rules exceed an image's capacity, the stored file is preserved and + policy saves are blocked. The CLI reports the capacity mismatch; install + firmware with enough rule slots to recover the complete policy. + ## Installation An ESP32 merged `.bin` installs bootloader, partitions and application over @@ -43,6 +52,11 @@ family's utility page, or the matching merged image over USB. Follow the ZIP's README and use its supplied Full application. `get storage.layout` reports the live layout on firmware that supports it. +Check the image's flood-rule capacity before updating a repeater. Move active +rules into the supported slots and save them before installing a smaller +table. An over-capacity policy is retained on storage but remains inactive +until compatible firmware is installed. + Native hardware CDC can change the USB product name and Linux `/dev/serial/by-id` path. Update serial consumers to the new device path after the first upgrade. Do not identify a radio solely by a changing `/dev/ttyACM` number. diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 84ed4feee..056287add 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -3536,6 +3536,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc #endif #if MESH_ENABLE_FLOOD_RULE_ENGINE flood_policy_has_embedded_sections = false; + flood_policy_capacity_limited = false; flood_channel_data_rule_slot = 0xFF; flood_channel_data_rule_max_hops = FLOOD_CHANNEL_HOPS_ALL; #endif @@ -6022,12 +6023,13 @@ static_assert(PUB_KEY_SIZE == FloodFilterPolicy::CHANNEL_KEY_256_LEN, bool MyMesh::loadFloodPacketFilters() { if (flood_packet_filter_slots == 0) return false; + flood_policy_capacity_limited = false; if (_fs == NULL) { seedDefaultFloodPacketFilters(); return true; } - enum class FileState : uint8_t { Missing, Valid, Invalid, Unreadable }; + enum class FileState : uint8_t { Missing, Valid, Invalid, Unreadable, CapacityExceeded }; auto loadFile = [this](const char* filename) -> FileState { if (flood_packet_filters) memset(flood_packet_filters, 0, sizeof(FloodPacketFilterEntry) * flood_packet_filter_slots); flood_channel_data_rule_slot = 0xFF; @@ -6051,11 +6053,22 @@ bool MyMesh::loadFloodPacketFilters() { bool version_6 = success && memcmp(magic, "FPF6", sizeof(magic)) == 0; bool version_7 = success && memcmp(magic, "FPF7", sizeof(magic)) == 0; success = (version_6 || version_7) - && file.read(&count, sizeof(count)) == sizeof(count) - && FloodFilterPolicy::forwardPersistenceCountSupported( - count, flood_packet_filter_slots); + && file.read(&count, sizeof(count)) == sizeof(count); + // A wider firmware image can write inactive trailing slots. Decode them + // through one bounded scratch entry so harmless padding remains compatible. + // Counts beyond the rule engine's mask may belong to newer firmware; retain + // that file instead of treating an unsupported capacity as corrupt bytes. + if (success && !FloodFilterPolicy::forwardPersistenceCountSupported(count, 64)) { + file.close(); + flood_policy_capacity_limited = true; + return FileState::CapacityExceeded; + } + FloodPacketFilterEntry overflow_entry; + bool active_overflow = false; for (int i = 0; success && i < count; i++) { + auto& entry = i < flood_packet_filter_slots ? loaded[i] : overflow_entry; + if (i >= flood_packet_filter_slots) memset(&entry, 0, sizeof(entry)); uint8_t active = 0; uint8_t suspend_on_temp_radio = 0; uint8_t match_blacklisted_path = 0; @@ -6066,17 +6079,17 @@ bool MyMesh::loadFloodPacketFilters() { uint8_t stop_on_match = 0; uint8_t stored_rule_channel = 0; success = file.read(&active, sizeof(active)) == sizeof(active); - success = success && file.read(&loaded[i].payload_type, sizeof(loaded[i].payload_type)) == sizeof(loaded[i].payload_type); - success = success && file.read(&loaded[i].min_hops, sizeof(loaded[i].min_hops)) == sizeof(loaded[i].min_hops); - success = success && file.read(&loaded[i].max_hops, sizeof(loaded[i].max_hops)) == sizeof(loaded[i].max_hops); + success = success && file.read(&entry.payload_type, sizeof(entry.payload_type)) == sizeof(entry.payload_type); + success = success && file.read(&entry.min_hops, sizeof(entry.min_hops)) == sizeof(entry.min_hops); + success = success && file.read(&entry.max_hops, sizeof(entry.max_hops)) == sizeof(entry.max_hops); success = success && file.read(&suspend_on_temp_radio, sizeof(suspend_on_temp_radio)) == sizeof(suspend_on_temp_radio); - success = success && file.read((uint8_t*)loaded[i].scope_name, - sizeof(loaded[i].scope_name)) - == sizeof(loaded[i].scope_name); + success = success && file.read((uint8_t*)entry.scope_name, + sizeof(entry.scope_name)) + == sizeof(entry.scope_name); success = success - && memchr(loaded[i].scope_name, 0, sizeof(loaded[i].scope_name)) != NULL; + && memchr(entry.scope_name, 0, sizeof(entry.scope_name)) != NULL; success = success && file.read(&match_blacklisted_path, sizeof(match_blacklisted_path)) == sizeof(match_blacklisted_path); @@ -6087,36 +6100,36 @@ bool MyMesh::loadFloodPacketFilters() { sizeof(scope_uses_slow_timing)) == sizeof(scope_uses_slow_timing); if (success && version_7) { - success = file.read(&loaded[i].incoming_scope_kind, - sizeof(loaded[i].incoming_scope_kind)) - == sizeof(loaded[i].incoming_scope_kind); + success = file.read(&entry.incoming_scope_kind, + sizeof(entry.incoming_scope_kind)) + == sizeof(entry.incoming_scope_kind); success = success - && file.read((uint8_t*)loaded[i].incoming_scope_name, - sizeof(loaded[i].incoming_scope_name)) - == sizeof(loaded[i].incoming_scope_name); + && file.read((uint8_t*)entry.incoming_scope_name, + sizeof(entry.incoming_scope_name)) + == sizeof(entry.incoming_scope_name); success = success && file.read(&stored_rule_channel, sizeof(stored_rule_channel)) == sizeof(stored_rule_channel); success = success - && file.read(loaded[i].channel_secret, - sizeof(loaded[i].channel_secret)) - == sizeof(loaded[i].channel_secret); + && file.read(entry.channel_secret, + sizeof(entry.channel_secret)) + == sizeof(entry.channel_secret); success = success - && file.read((uint8_t*)loaded[i].channel_name, - sizeof(loaded[i].channel_name)) - == sizeof(loaded[i].channel_name); + && file.read((uint8_t*)entry.channel_name, + sizeof(entry.channel_name)) + == sizeof(entry.channel_name); success = success - && file.read(&loaded[i].path_hash_size, - sizeof(loaded[i].path_hash_size)) - == sizeof(loaded[i].path_hash_size); + && file.read(&entry.path_hash_size, + sizeof(entry.path_hash_size)) + == sizeof(entry.path_hash_size); success = success - && file.read(&loaded[i].path_hops, - sizeof(loaded[i].path_hops)) - == sizeof(loaded[i].path_hops); + && file.read(&entry.path_hops, + sizeof(entry.path_hops)) + == sizeof(entry.path_hops); success = success - && file.read(loaded[i].path, sizeof(loaded[i].path)) - == sizeof(loaded[i].path); + && file.read(entry.path, sizeof(entry.path)) + == sizeof(entry.path); success = success && file.read(&drop_on_match, sizeof(drop_on_match)) == sizeof(drop_on_match); @@ -6124,45 +6137,45 @@ bool MyMesh::loadFloodPacketFilters() { && file.read(&rate_limit_enabled, sizeof(rate_limit_enabled)) == sizeof(rate_limit_enabled); success = success - && file.read((uint8_t*)&loaded[i].rate_per_minute, - sizeof(loaded[i].rate_per_minute)) - == sizeof(loaded[i].rate_per_minute); + && file.read((uint8_t*)&entry.rate_per_minute, + sizeof(entry.rate_per_minute)) + == sizeof(entry.rate_per_minute); success = success - && file.read((uint8_t*)loaded[i].target_region_name, - sizeof(loaded[i].target_region_name)) - == sizeof(loaded[i].target_region_name); + && file.read((uint8_t*)entry.target_region_name, + sizeof(entry.target_region_name)) + == sizeof(entry.target_region_name); success = success - && file.read(&loaded[i].priority, sizeof(loaded[i].priority)) - == sizeof(loaded[i].priority); + && file.read(&entry.priority, sizeof(entry.priority)) + == sizeof(entry.priority); success = success && file.read(&stop_on_match, sizeof(stop_on_match)) == sizeof(stop_on_match); } else { - loaded[i].incoming_scope_kind = scope_requires_region_match + entry.incoming_scope_kind = scope_requires_region_match ? FloodFilterPolicy::RULE_IN_ALLOWED : FloodFilterPolicy::RULE_IN_ANY; - drop_on_match = loaded[i].scope_name[0] == 0 ? 1 : 0; + drop_on_match = entry.scope_name[0] == 0 ? 1 : 0; } if (success && version_7) { success = FloodFilterPolicy::decodeStoredRuleChannel( - stored_rule_channel, loaded[i].channel_key_len, - loaded[i].retry_on_match); + stored_rule_channel, entry.channel_key_len, + entry.retry_on_match); } else { - loaded[i].retry_on_match = false; + entry.retry_on_match = false; } if (version_7) { success = success && FloodFilterPolicy::decodeStoredRuleActive( - active, loaded[i].active, loaded[i].transport_modes); + active, entry.active, entry.transport_modes); } else { - loaded[i].active = active != 0; + entry.active = active != 0; success = success && active <= 1; } - loaded[i].suspend_on_temp_radio = suspend_on_temp_radio != 0; - loaded[i].match_blacklisted_path = match_blacklisted_path != 0; - loaded[i].scope_uses_slow_timing = scope_uses_slow_timing != 0; - loaded[i].drop_on_match = drop_on_match != 0; - loaded[i].rate_limit_enabled = rate_limit_enabled != 0; - loaded[i].stop_on_match = stop_on_match != 0; + entry.suspend_on_temp_radio = suspend_on_temp_radio != 0; + entry.match_blacklisted_path = match_blacklisted_path != 0; + entry.scope_uses_slow_timing = scope_uses_slow_timing != 0; + entry.drop_on_match = drop_on_match != 0; + entry.rate_limit_enabled = rate_limit_enabled != 0; + entry.stop_on_match = stop_on_match != 0; if (success && (suspend_on_temp_radio > 1 || match_blacklisted_path > 1 || scope_requires_region_match > 1 || scope_uses_slow_timing > 1 || drop_on_match > 1 @@ -6170,105 +6183,116 @@ bool MyMesh::loadFloodPacketFilters() { success = false; } success = success && FloodFilterPolicy::transportActionsSupported( - loaded[i].transport_modes, loaded[i].scope_name[0] != 0 - || loaded[i].target_region_name[0] != 0, - loaded[i].retry_on_match, loaded[i].scope_uses_slow_timing); + entry.transport_modes, entry.scope_name[0] != 0 + || entry.target_region_name[0] != 0, + entry.retry_on_match, entry.scope_uses_slow_timing); if (!success) break; - if (!loaded[i].active) { - memset(&loaded[i], 0, sizeof(loaded[i])); + if (!entry.active) { + memset(&entry, 0, sizeof(entry)); continue; } - bool direct_target = loaded[i].scope_name[0] != 0; + bool direct_target = entry.scope_name[0] != 0; bool target_name_terminated = memchr( - loaded[i].target_region_name, 0, - sizeof(loaded[i].target_region_name)) != NULL; + entry.target_region_name, 0, + sizeof(entry.target_region_name)) != NULL; bool region_target = target_name_terminated - && loaded[i].target_region_name[0] != 0; + && entry.target_region_name[0] != 0; bool input_name_terminated = memchr( - loaded[i].incoming_scope_name, 0, - sizeof(loaded[i].incoming_scope_name)) != NULL; + entry.incoming_scope_name, 0, + sizeof(entry.incoming_scope_name)) != NULL; bool channel_name_terminated = memchr( - loaded[i].channel_name, 0, sizeof(loaded[i].channel_name)) != NULL; - bool incoming_valid = loaded[i].incoming_scope_kind + entry.channel_name, 0, sizeof(entry.channel_name)) != NULL; + bool incoming_valid = entry.incoming_scope_kind <= FloodFilterPolicy::RULE_IN_REGION; if (incoming_valid - && loaded[i].incoming_scope_kind == FloodFilterPolicy::RULE_IN_SCOPE) { + && entry.incoming_scope_kind == FloodFilterPolicy::RULE_IN_SCOPE) { incoming_valid = input_name_terminated && isValidStoredFloodFilterScopeName( - loaded[i].incoming_scope_name); + entry.incoming_scope_name); } else if (incoming_valid - && loaded[i].incoming_scope_kind == FloodFilterPolicy::RULE_IN_REGION) { + && entry.incoming_scope_kind == FloodFilterPolicy::RULE_IN_REGION) { incoming_valid = input_name_terminated && isValidStoredFloodRuleRegionName( - loaded[i].incoming_scope_name); + entry.incoming_scope_name); } else if (incoming_valid) { incoming_valid = input_name_terminated - && loaded[i].incoming_scope_name[0] == 0; + && entry.incoming_scope_name[0] == 0; } bool channel_valid = FloodFilterPolicy::channelKeyLengthSupported( - loaded[i].channel_key_len); - if (channel_valid && loaded[i].channel_key_len == 0) { + entry.channel_key_len); + if (channel_valid && entry.channel_key_len == 0) { channel_valid = channel_name_terminated - && loaded[i].channel_name[0] == 0; + && entry.channel_name[0] == 0; } else if (channel_valid && FloodFilterPolicy::channelHashOnly( - loaded[i].channel_key_len)) { + entry.channel_key_len)) { channel_valid = channel_name_terminated; if (channel_valid) { - loaded[i].channel_hash = loaded[i].channel_secret[0]; - memset(&loaded[i].channel_secret[1], 0, - sizeof(loaded[i].channel_secret) - 1); - snprintf(loaded[i].channel_name, - sizeof(loaded[i].channel_name), "hash:%02X", - loaded[i].channel_hash); + entry.channel_hash = entry.channel_secret[0]; + memset(&entry.channel_secret[1], 0, + sizeof(entry.channel_secret) - 1); + snprintf(entry.channel_name, + sizeof(entry.channel_name), "hash:%02X", + entry.channel_hash); } } else if (channel_valid) { channel_valid = channel_name_terminated - && loaded[i].channel_name[0] != 0; + && entry.channel_name[0] != 0; if (channel_valid) { - mesh::Utils::sha256(&loaded[i].channel_hash, - sizeof(loaded[i].channel_hash), - loaded[i].channel_secret, - loaded[i].channel_key_len); + mesh::Utils::sha256(&entry.channel_hash, + sizeof(entry.channel_hash), + entry.channel_secret, + entry.channel_key_len); } } - if (channel_valid && loaded[i].channel_key_len != 0) { - channel_valid = loaded[i].payload_type == FLOOD_PACKET_FILTER_ANY_TYPE - || loaded[i].payload_type == PAYLOAD_TYPE_GRP_TXT - || loaded[i].payload_type == PAYLOAD_TYPE_GRP_DATA; + if (channel_valid && entry.channel_key_len != 0) { + channel_valid = entry.payload_type == FLOOD_PACKET_FILTER_ANY_TYPE + || entry.payload_type == PAYLOAD_TYPE_GRP_TXT + || entry.payload_type == PAYLOAD_TYPE_GRP_DATA; } bool path_valid = FloodFilterPolicy::pathMatcherValid( - loaded[i].path_hash_size, loaded[i].path_hops, - loaded[i].match_blacklisted_path); - bool action_valid = loaded[i].drop_on_match || direct_target - || region_target || loaded[i].rate_limit_enabled - || loaded[i].stop_on_match || loaded[i].retry_on_match; - if (!((loaded[i].payload_type <= PH_TYPE_MASK - || loaded[i].payload_type == FLOOD_PACKET_FILTER_ANY_TYPE) - && loaded[i].min_hops <= loaded[i].max_hops - && loaded[i].max_hops <= FLOOD_PACKET_FILTER_MAX_HOPS + entry.path_hash_size, entry.path_hops, + entry.match_blacklisted_path); + bool action_valid = entry.drop_on_match || direct_target + || region_target || entry.rate_limit_enabled + || entry.stop_on_match || entry.retry_on_match; + if (!((entry.payload_type <= PH_TYPE_MASK + || entry.payload_type == FLOOD_PACKET_FILTER_ANY_TYPE) + && entry.min_hops <= entry.max_hops + && entry.max_hops <= FLOOD_PACKET_FILTER_MAX_HOPS && (!direct_target - || isValidStoredFloodFilterScopeName(loaded[i].scope_name)) + || isValidStoredFloodFilterScopeName(entry.scope_name)) && target_name_terminated && (!region_target || isValidStoredFloodRuleRegionName( - loaded[i].target_region_name)) + entry.target_region_name)) && !(direct_target && region_target) - && !(loaded[i].drop_on_match && (direct_target || region_target)) - && !(loaded[i].drop_on_match && loaded[i].rate_limit_enabled) - && !(loaded[i].drop_on_match && loaded[i].retry_on_match) - && (!loaded[i].rate_limit_enabled - || loaded[i].rate_per_minute + && !(entry.drop_on_match && (direct_target || region_target)) + && !(entry.drop_on_match && entry.rate_limit_enabled) + && !(entry.drop_on_match && entry.retry_on_match) + && (!entry.rate_limit_enabled + || entry.rate_per_minute < FLOOD_GROUP_MODERATION_RATE_UNLIMITED) - && (!loaded[i].scope_uses_slow_timing + && (!entry.scope_uses_slow_timing || direct_target || region_target) && incoming_valid && channel_valid && path_valid && action_valid)) { success = false; } + if (success && i >= flood_packet_filter_slots) active_overflow = true; + } + + if (success && active_overflow) { + // Do not activate only part of an operator's policy or discard the stored + // transaction. A larger-capacity image can read it without data loss. + memset(flood_packet_filters, 0, + sizeof(FloodPacketFilterEntry) * flood_packet_filter_slots); + file.close(); + flood_policy_capacity_limited = true; + return FileState::CapacityExceeded; } // FPF7 keeps the forwarding rows byte-compatible with the room-server @@ -6379,6 +6403,7 @@ bool MyMesh::loadFloodPacketFilters() { if (success && file.available() != 0) success = false; if (success && compatibility_slot != 0xFF) { success = compatibility_slot < count + && compatibility_slot < flood_packet_filter_slots && isFloodChannelDataRule(loaded[compatibility_slot]) && loaded[compatibility_slot].min_hops == (compatibility_max_hops == FLOOD_CHANNEL_HOPS_ALL @@ -6408,6 +6433,7 @@ bool MyMesh::loadFloodPacketFilters() { }; FileState primary = loadFile(FLOOD_PACKET_FILTER_FILE); + if (primary == FileState::CapacityExceeded) return false; if (primary == FileState::Valid) { // A valid primary is already committed. Transaction remnants are stale. if (_fs->exists(FLOOD_PACKET_FILTER_TEMP_FILE)) @@ -6423,6 +6449,7 @@ bool MyMesh::loadFloodPacketFilters() { } FileState temp = loadFile(FLOOD_PACKET_FILTER_TEMP_FILE); + if (temp == FileState::CapacityExceeded) return false; if (temp == FileState::Valid) { // A complete temp is the newest transaction image. Never destroy an // unreadable primary, but still use the verified temp in RAM this boot. @@ -6440,6 +6467,7 @@ bool MyMesh::loadFloodPacketFilters() { } FileState backup = loadFile(FLOOD_PACKET_FILTER_BACKUP_FILE); + if (backup == FileState::CapacityExceeded) return false; if (backup == FileState::Valid) { if (primary != FileState::Unreadable) { if (primary == FileState::Invalid) @@ -6821,6 +6849,7 @@ bool MyMesh::migrateLegacyFloodChannelBlocks() { bool MyMesh::saveFloodPacketFilters(bool empty_scope_phase, bool empty_forward_phase) { if (_fs == NULL) return false; + if (flood_policy_capacity_limited) return false; // Without a rule table there is nothing to persist. Writing the file anyway // would replace the operator's stored ruleset with an empty one. if (flood_packet_filter_slots == 0) return false; @@ -7609,6 +7638,12 @@ void MyMesh::formatFloodPacketFilterDetail(int index, char* reply, size_t reply_ } void MyMesh::formatFloodPacketFilters(const char* args, char* reply, bool compact) const { + if (flood_policy_capacity_limited) { + snprintf(reply, 160, + "Err - saved rules exceed this image's %u slots; install an image with more rule slots", + (unsigned)flood_packet_filter_slots); + return; + } const char* selector = skipFloodFilterSpaces(args); if (*selector == '.') selector = skipFloodFilterSpaces(selector + 1); if (*selector != 0) { diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 9e2ca7370..d0e871a2c 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -506,6 +506,9 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks uint64_t recv_pkt_filter_match_mask; #if MESH_ENABLE_FLOOD_RULE_ENGINE bool flood_policy_has_embedded_sections; + // A valid stored ruleset can exceed this image's runtime table. Preserve + // it until compatible firmware is installed; do not publish an empty set. + bool flood_policy_capacity_limited; // Zero-based forward-row slot owned by the flood.channel.data compatibility // facade. 0xFF means forwarding is enabled and no compatibility row exists. uint8_t flood_channel_data_rule_slot; diff --git a/test/test_build_size_detection.sh b/test/test_build_size_detection.sh index d8f38b1c3..e67d8e8a8 100644 --- a/test/test_build_size_detection.sh +++ b/test/test_build_size_detection.sh @@ -113,7 +113,23 @@ for target in Heltec_t096_repeater_lora_ota_no_external_sensors Heltec_t1_repeat [[ "$PLATFORMIO_BUILD_FLAGS" == *'-DFLOOD_PACKET_FILTER_SLOTS=4'* ]] || fail "$target omitted RAM-safe rule table" [[ "$PLATFORMIO_BUILD_FLAGS" != *'MESH_MIN_RUNTIME_HEAP'* ]] || fail "runtime RAM guard was overridden" done -for target in Heltec_t096_companion_radio_full_femon RAK_4631_repeater; do +# Unified internal/external RAK3401 staging reserves the 64 KiB internal +# arena even with QSPI available. Keep the stack, sensors and table engine. +( + target=RAK_3401_repeater_unified_lora_ota + PIO_ENV_PLATFORM_BY_NAME[$target]=NRF52_PLATFORM + for sensor_profile in full reduced; do + NRF52_OTA_SENSOR_PROFILE=$sensor_profile + PLATFORMIO_BUILD_FLAGS='-DENABLE_OTA=1 -DOTA_FLASH_STORE=1 -DOTA_QSPI_STORE=1 -DMAX_NEIGHBOURS=50' + PLATFORMIO_BUILD_UNFLAGS='' + apply_nrf52_size_profile "$target" + [[ "$PLATFORMIO_BUILD_FLAGS" == *'-DFLOOD_PACKET_FILTER_SLOTS=47'* ]] || fail "$target omitted safe unified-storage rule capacity" + [[ "$PLATFORMIO_BUILD_FLAGS" == *'-DMAX_NEIGHBOURS=50'* ]] || fail "$target changed neighbors" + [[ "$PLATFORMIO_BUILD_FLAGS" != *'MESH_NRF52_LOOP_STACK_WORDS'* && "$PLATFORMIO_BUILD_FLAGS" != *'MESH_MIN_RUNTIME_HEAP'* ]] || fail "$target weakened the stack or runtime RAM guard" + [[ "$PLATFORMIO_BUILD_UNFLAGS" != *'ENV_INCLUDE_'* && "$PLATFORMIO_BUILD_UNFLAGS" != *'OTA_FLASH_STORE'* && "$PLATFORMIO_BUILD_UNFLAGS" != *'OTA_QSPI_STORE'* ]] || fail "$target removed sensors or storage" + done +) +for target in Heltec_t096_companion_radio_full_femon RAK_4631_repeater RAK_4631_repeater_unified_lora_ota RAK_3401_repeater_rak13302_w25q16_lora_ota; do PIO_ENV_PLATFORM_BY_NAME[$target]=NRF52_PLATFORM PLATFORMIO_BUILD_FLAGS='' PLATFORMIO_BUILD_UNFLAGS='' diff --git a/test/test_firmware_picker.js b/test/test_firmware_picker.js index f23f6d94a..18ecd9c5a 100644 --- a/test/test_firmware_picker.js +++ b/test/test_firmware_picker.js @@ -194,6 +194,27 @@ assert(!releaseSet.releases.some(function (item) { return item.tag_name.startsWith("nrf52-mota-"); })); +// Numbered pages are emitted when a category exceeds the asset limit. +// Include them only for the exact family and known category names. +const chunkTags = ["companion-2-", "lora-ota-2-", "full-profiles-10-"]; +const chunkedReleases = releases.concat(chunkTags.map(function (prefix, index) { + return release(prefix + family, "2026-08-23T12:00:00Z", [ + asset("Extra" + index + "_repeater-" + family + ".uf2") + ]); +})); +for (const prefix of ["companion-1-", "lora-ota-0-", "lora-ota-02-", "unknown-2-"]) { + chunkedReleases.push(release(prefix + family, "2026-08-23T12:00:00Z", [])); +} +chunkedReleases.push(release("lora-ota-2-" + family + "-other", "2026-08-23T12:00:00Z", [])); +chunkedReleases.push(release("lora-ota-3-" + family, "2026-08-23T12:00:00Z", [], { draft: true })); +const chunkedSet = picker.selectReleaseSet(chunkedReleases); +assert.strictEqual(chunkedSet.releases.length, releaseSet.releases.length + chunkTags.length); +for (const prefix of chunkTags) assert(chunkedSet.releases.some(r => r.tag_name === prefix + family)); +const chunkedCatalog = picker.buildCatalog(chunkedReleases); +for (let index = 0; index < chunkTags.length; ++index) { + assert(chunkedCatalog.profiles.some(p => p.target === "Extra" + index + "_repeater")); +} + const catalog = picker.buildCatalog(releases); assert.strictEqual(catalog.releaseSet.familyTag, family); assert.strictEqual(catalog.profiles.length, 12); diff --git a/test/test_flood_policy_capacity.py b/test/test_flood_policy_capacity.py new file mode 100644 index 000000000..030dbefb1 --- /dev/null +++ b/test/test_flood_policy_capacity.py @@ -0,0 +1,231 @@ +"""Run production flood-policy load/save code across firmware capacity changes.""" +from pathlib import Path +import os +import subprocess +import tempfile +import unittest +from test_common_prefs_commit import HARNESS +from test_replay_reset_integration import extract_braced + +ROOT = Path(__file__).resolve().parents[1] + +PRELUDE = r''' +#include +#include +#include +#include +#define MESH_ENABLE_FLOOD_RULE_ENGINE 1 +#define FLOOD_PACKET_FILTER_SCOPE_NAME_LEN 32 +#define FLOOD_GROUP_MODERATION_NAME_LEN 32 +#define FLOOD_PACKET_FILTER_PATH_PREFIX_BYTES_MAX 9 +#define FLOOD_PACKET_FILTER_ANY_TYPE 0xFF +#define FLOOD_PACKET_FILTER_MAX_HOPS 63 +#define FLOOD_CHANNEL_HOPS_ALL 0xFF +#define FLOOD_GROUP_MODERATION_RATE_UNLIMITED 0xFFFF +#define FLOOD_CHANNEL_SCOPE_OTHER_ANY 2 +#define FLOOD_CHANNEL_SCOPE_SLOTS 15 +#define FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS 15 +#define FLOOD_PACKET_FILTER_BLACKLIST_MAX 18 +#define FLOOD_PACKET_FILTER_PATH_ID_SIZE 3 +#define FLOOD_PACKET_FILTER_FILE "/flood_filter" +#define FLOOD_PACKET_FILTER_TEMP_FILE "/flood_filter.tmp" +#define FLOOD_PACKET_FILTER_BACKUP_FILE "/flood_filter.bak" +#define FLOOD_POLICY_SECTION_MAGIC "FPS1" +#define FILE_O_WRITE "w" +namespace mesh { +File openFileRead(FILESYSTEM* fs, const char* path) { return fs->open(path); } +struct Utils { + static void sha256(void*, size_t, const uint8_t*, size_t) { + assert(false && "crypto is outside these capacity fixtures"); + } +}; +} +struct RegionMap { + static bool is_name_char(uint8_t c) { return std::isalnum(c) || c=='_' || c=='-'; } +}; +struct StrHelper { + static void strzcpy(char* out, const char* in, size_t size) { + if (out!=in) std::memmove(out,in,std::min(size,std::strlen(in)+1)); + if (size) out[size-1]=0; + } +}; +''' +CLASS = r''' +class MyMesh { +public: +@ENTRY@; +@SCOPE@; + FILESYSTEM* _fs; + std::vector storage; + FloodPacketFilterEntry* flood_packet_filters; + uint8_t flood_packet_filter_slots; + bool flood_policy_has_embedded_sections = false; + bool flood_policy_capacity_limited = false; + uint8_t flood_channel_data_rule_slot = 0xff; + uint8_t flood_channel_data_rule_max_hops = FLOOD_CHANNEL_HOPS_ALL; + FloodChannelScopeEntry flood_channel_scopes[FLOOD_CHANNEL_SCOPE_SLOTS]{}; + char flood_channel_direct_scopes[FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS][32]{}; + uint8_t flood_packet_filter_blacklist_count = 0; + uint8_t flood_packet_filter_blacklist[18][3]{}; + unsigned seeded = 0; + MyMesh(MemoryFS& fs, uint8_t slots): _fs(&fs),storage(slots), + flood_packet_filters(storage.data()),flood_packet_filter_slots(slots) {} + void seedDefaultFloodPacketFilters() { ++seeded; } + bool loadFloodPacketFilters(); + bool saveFloodPacketFilters(bool empty_scope_phase=false,bool empty_forward_phase=false); + bool isFloodChannelDataRule(const FloodPacketFilterEntry&) const; +}; +''' +SCENARIOS = r''' +std::vector fixture(uint8_t slots, uint8_t highest) { + MemoryFS fs;MyMesh writer(fs,slots); + for (uint8_t index : {uint8_t(0),highest}) { + auto& row=writer.storage[index];row.active=true;row.payload_type=PAYLOAD_TYPE_ADVERT; + row.min_hops=2;row.max_hops=63;row.drop_on_match=true; + row.incoming_scope_kind=FloodFilterPolicy::RULE_IN_ANY; + row.transport_modes=FloodFilterPolicy::RULE_MODE_RADIO; + } + assert(writer.saveFloodPacketFilters()); + const auto bytes=fs.get(FLOOD_PACKET_FILTER_FILE); + MyMesh same(fs,slots);assert(same.loadFloodPacketFilters()); + assert(same.storage[highest].active && same.storage[highest].drop_on_match); + return bytes; +} +std::map> snapshot(const MemoryFS& fs) { + std::map> result; + for (const auto& row:fs.files) result[row.first]=*row.second; + return result; +} +int main() { + const auto large=fixture(63,62),small=fixture(4,2); + // Older writers stored every slot, including inactive trailing entries. + // Build the exact production wire format and replace the final active row + // with its serialized empty neighbor; the highest retained rule is slot 1. + MemoryFS empty_fs;MyMesh empty(empty_fs,63);assert(empty.saveFloodPacketFilters()); + const auto overhead=empty_fs.get(FLOOD_PACKET_FILTER_FILE).size(); + assert((large.size()-overhead)%63==0); + const size_t stride=(large.size()-overhead)/63; + auto padded=large; + std::copy_n(large.begin()+5+stride,stride,padded.begin()+5+62*stride); + for (uint8_t capacity : {uint8_t(4),uint8_t(47)}) { + MemoryFS fs;fs.put(FLOOD_PACKET_FILTER_FILE,padded);MyMesh reader(fs,capacity); + assert(reader.loadFloodPacketFilters());assert(reader.storage[0].active); + assert(!reader.flood_policy_capacity_limited && reader.seeded==0); + assert(fs.get(FLOOD_PACKET_FILTER_FILE)==padded); + assert(reader.saveFloodPacketFilters()); + assert(fs.get(FLOOD_PACKET_FILTER_FILE)[4]==1); + } + // FPF6 used a fixed 40-byte legacy row and also stored inactive tails. + for (bool active_overflow : {false,true}) { + std::vector legacy={'F','P','F','6',63}; + for (unsigned index=0;index<63;++index) { + std::vector row(40,0); + if(index==0 || (active_overflow && index==62)) { + row[0]=1;row[1]=PAYLOAD_TYPE_ADVERT;row[2]=2;row[3]=63; + } + legacy.insert(legacy.end(),row.begin(),row.end()); + } + MemoryFS fs;fs.put(FLOOD_PACKET_FILTER_FILE,legacy);MyMesh reader(fs,4); + assert(reader.loadFloodPacketFilters()==!active_overflow); + assert(reader.flood_policy_capacity_limited==active_overflow); + assert(reader.storage[0].active==!active_overflow); + assert(fs.get(FLOOD_PACKET_FILTER_FILE)==legacy); + } + // An extension may not reference an inactive row beyond the new table. + // This malformed padded image must not perform an out-of-bounds read. + { + auto malformed=padded; + assert(std::memcmp(malformed.data()+5+63*stride,"FPS1",4)==0); + malformed[5+63*stride+4]=62; + MemoryFS fs;fs.put(FLOOD_PACKET_FILTER_FILE,malformed);MyMesh reader(fs,4); + assert(reader.loadFloodPacketFilters());assert(reader.seeded==1); + assert(!fs.exists(FLOOD_PACKET_FILTER_FILE)); + } + // Future counts outside the 64-bit rule mask remain untouched as unsupported. + { + MemoryFS fs;const std::vector future={'F','P','F','7',65}; + fs.put(FLOOD_PACKET_FILTER_FILE,future);MyMesh reader(fs,4); + assert(!reader.loadFloodPacketFilters() && reader.flood_policy_capacity_limited); + assert(!reader.saveFloodPacketFilters());assert(fs.get(FLOOD_PACKET_FILTER_FILE)==future); + } + // Active overflow must never be mistaken for corrupt storage. Preserve all + // transaction images and refuse saves, including the empty-table path. + for (uint8_t capacity : {uint8_t(4),uint8_t(47)}) { + for (unsigned source=0;source<4;++source) { + MemoryFS fs; + if (source==0) { + fs.put(FLOOD_PACKET_FILTER_FILE,large); + fs.put(FLOOD_PACKET_FILTER_TEMP_FILE,small); + fs.put(FLOOD_PACKET_FILTER_BACKUP_FILE,small); + } else if(source==1) { + fs.put(FLOOD_PACKET_FILTER_TEMP_FILE,large); + fs.put(FLOOD_PACKET_FILTER_BACKUP_FILE,small); + } else if(source==2) { + fs.put(FLOOD_PACKET_FILTER_BACKUP_FILE,large); + } else { + fs.put(FLOOD_PACKET_FILTER_FILE,{0,1,2}); + fs.put(FLOOD_PACKET_FILTER_TEMP_FILE,large); + fs.put(FLOOD_PACKET_FILTER_BACKUP_FILE,small); + } + const auto before=snapshot(fs);MyMesh reader(fs,capacity); + assert(!reader.loadFloodPacketFilters()); + assert(reader.flood_policy_capacity_limited && reader.seeded==0); + assert(std::none_of(reader.storage.begin(),reader.storage.end(),[](const auto& e){return e.active;})); + assert(!reader.saveFloodPacketFilters()); + assert(!reader.saveFloodPacketFilters(true,true)); + assert(snapshot(fs)==before); + MyMesh compatible(fs,63);assert(compatible.loadFloodPacketFilters()); + assert(compatible.storage[62].active); + } + } + // Normal committed and recoverable tables still use the production path. + for (const char* path : {FLOOD_PACKET_FILTER_FILE,FLOOD_PACKET_FILTER_TEMP_FILE,FLOOD_PACKET_FILTER_BACKUP_FILE}) { + MemoryFS fs;fs.put(path,small);MyMesh reader(fs,4); + assert(reader.loadFloodPacketFilters());assert(reader.storage[2].active); + assert(!reader.flood_policy_capacity_limited); + assert(fs.get(FLOOD_PACKET_FILTER_FILE)==small); + } + // Malformed recognized images remain corrupt, unlike a capacity mismatch. + MemoryFS bad;bad.put(FLOOD_PACKET_FILTER_FILE,{0,1,2});MyMesh reader(bad,4); + assert(reader.loadFloodPacketFilters());assert(reader.seeded==1); + assert(!reader.flood_policy_capacity_limited); + assert(!bad.exists(FLOOD_PACKET_FILTER_FILE)); +} +''' + + +class FloodPolicyCapacityTests(unittest.TestCase): + def test_saved_rules_survive_smaller_firmware_capacity(self): + source=(ROOT/'examples/simple_repeater/MyMesh.cpp').read_text() + header=(ROOT/'examples/simple_repeater/MyMesh.h').read_text() + fs=HARNESS[:HARNESS.index('@STORE@')] + fs=fs.replace('bool valid = false;', 'bool valid = false;\n size_t cursor=0;') + fs=fs.replace('size_t write(const uint8_t*, size_t);', '''size_t write(const uint8_t*, size_t); + size_t read(uint8_t* out,size_t count) { + if (!valid) return 0; + count=std::min(count,bytes->size()-cursor); + std::copy_n(bytes->data()+cursor,count,out);cursor+=count;return count; + } + size_t available() const { return valid ? bytes->size()-cursor : 0; } +''') + fs='#include \n'+fs + parts=[fs,PRELUDE,CLASS.replace('@ENTRY@',extract_braced(header,'struct FloodPacketFilterEntry')).replace('@SCOPE@',extract_braced(header,'struct FloodChannelScopeEntry'))] + for marker in ('static File openFloodSettingsRead(', 'static File openFloodSettingsWrite(', + 'static uint32_t updateFloodSettingsHash(', 'static bool verifyFloodSettingsWrite(', + 'static bool isValidStoredFloodFilterScopeName(', 'static bool isValidStoredFloodRuleRegionName(', + 'bool MyMesh::isFloodChannelDataRule(', 'bool MyMesh::loadFloodPacketFilters(', + 'bool MyMesh::saveFloodPacketFilters('): + parts.append(extract_braced(source,marker)) + parts.append(SCENARIOS) + with tempfile.TemporaryDirectory(prefix='flood-policy-capacity-') as temp: + p=Path(temp);cpp=p/'test.cpp';cpp.write_text('\n'.join(parts)) + for platform in ('NRF52_PLATFORM','ESP32'): + with self.subTest(platform=platform): + exe=p/platform + subprocess.run(['g++','-std=c++17','-O1','-g','-fsanitize=address,undefined', + '-fno-omit-frame-pointer','-no-pie','-D'+platform, + '-I'+str(ROOT/'src'),str(cpp),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True,env={**os.environ,'ASAN_OPTIONS':'detect_leaks=1'}) + + +if __name__=='__main__':unittest.main()