diff --git a/examples/companion_radio/DataStore.cpp b/examples/companion_radio/DataStore.cpp index f3f8b3369..549a582e6 100644 --- a/examples/companion_radio/DataStore.cpp +++ b/examples/companion_radio/DataStore.cpp @@ -64,8 +64,9 @@ static File openWrite(FILESYSTEM* fs, const char* filename) { static bool validateLfsFilesystem(FILESYSTEM* fs); #endif #if defined(NRF52_PLATFORM) -static bool recoverPrimaryFilesystem(FILESYSTEM* fs); static void cleanupAtomicTempFiles(FILESYSTEM* fs); +static bool contactPathPresence(FILESYSTEM* fs, const char* path, + bool& present, uint32_t* size = nullptr); #endif void DataStore::begin() { @@ -75,28 +76,58 @@ void DataStore::begin() { #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) #if defined(NRF52_PLATFORM) + resetContactPageState(true); + if (_secondary_authority_unknown) { + _contact_load_incomplete = true; + _prefs_load_incomplete = true; + } + if (_primary_storage_unavailable) { + // The base mount already failed. Do not traverse or open this unmounted + // filesystem; both can re-enter corrupt metadata before explicit repair. + _identity_creation_blocked = true; + _contact_load_incomplete = true; + _fsExtra = nullptr; + return; + } + bool primary_ready = validateLfsFilesystem(_fs); if (!primary_ready) { - MESH_DEBUG_PRINTLN("DataStore: primary LittleFS metadata is corrupt; rebuilding before first write"); - primary_ready = recoverPrimaryFilesystem(_fs); + // A traversal error can be transient, and formatting here would erase the + // only identity before MyMesh can distinguish recovery from a fresh boot. + // Preserve the filesystem and fail closed; a reboot may recover it, while + // an explicit factory reset remains the destructive recovery operation. + MESH_DEBUG_PRINTLN( + "DataStore: primary LittleFS metadata is unavailable; preserving it and blocking startup writes"); + _primary_storage_unavailable = true; + _identity_creation_blocked = true; + _contact_load_incomplete = true; + _prefs_load_incomplete = true; + _fsExtra = nullptr; + return; } if (_fsExtra != nullptr && !validateLfsFilesystem(_fsExtra)) { // A traversal-failed filesystem may still be recoverable offline. Keep // both removable QSPI and managed on-chip ExtraFS intact until an explicit // local repair/factory-reset operation authorizes erasing it. MESH_DEBUG_PRINTLN("DataStore: secondary LittleFS metadata is corrupt; preserving it and using primary storage"); - _fsExtra = nullptr; + disableSecondaryFS(true); } if (primary_ready) cleanupAtomicTempFiles(_fs); if (_fsExtra != nullptr) cleanupAtomicTempFiles(_fsExtra); - resetContactPageState(); #endif #if defined(EXTRAFS) || defined(QSPIFLASH) if (_fsExtra != nullptr && !migrateToSecondaryFS()) { MESH_DEBUG_PRINTLN("DataStore: one or more secondary filesystem migrations remain pending"); } #endif +#if defined(NRF52_PLATFORM) + // A journal-stat failure leaves the contact/channel authority unknowable. + // Do not mutate the reconstructable advert cache on either candidate store + // while that fail-closed latch is active. + if (!_contact_load_incomplete) checkAdvBlobFile(); +#else checkAdvBlobFile(); +#endif #else // init 'blob store' support _fs->mkdir("/bl"); @@ -153,20 +184,6 @@ static bool validateLfsFilesystem(FILESYSTEM* fs) { #endif #if defined(NRF52_PLATFORM) -static bool recoverPrimaryFilesystem(FILESYSTEM* fs) { - // Once traversal has found a bad pointer or metadata cycle, even read-only - // path lookup on this mounted filesystem can enter the same unbounded walk. - // Do not try to copy identity/preferences out through the corrupted metadata: - // rebuild immediately so the next mutation cannot hard-fault or hang forever. - const bool formatted = fs->format(); - if (!formatted) { - MESH_DEBUG_PRINTLN("DataStore: primary LittleFS rebuild failed"); - } else { - MESH_DEBUG_PRINTLN("DataStore: primary LittleFS rebuilt; corrupt contents discarded"); - } - return formatted; -} - static void cleanupAtomicTempFiles(FILESYSTEM* fs) { if (fs == nullptr) return; @@ -249,6 +266,32 @@ bool DataStore::reinitializeInternalExtraFS() { #endif #endif +void DataStore::markPrimaryFSUnavailable() { +#if defined(NRF52_PLATFORM) + _primary_storage_unavailable = true; + _identity_creation_blocked = true; + _contact_load_incomplete = true; + _prefs_load_incomplete = true; +#endif +} + +void DataStore::disableSecondaryFS(bool authority_unknown) { + _fsExtra = nullptr; +#if defined(NRF52_PLATFORM) + if (authority_unknown) { + // Without reading the secondary journal, primary may be a retired source + // rather than an empty fresh-install store. Keep this fact across begin()'s + // runtime-state reset and block every authority-changing write. + _secondary_authority_unknown = true; + _contact_load_incomplete = true; + _identity_creation_blocked = true; + _prefs_load_incomplete = true; + } +#else + (void)authority_unknown; +#endif +} + uint32_t DataStore::getStorageUsedKb() const { #if defined(ESP32) return SPIFFS.usedBytes() / 1024; @@ -328,7 +371,17 @@ bool DataStore::formatFileSystem() { #else const bool secondary_success = _fsExtra == nullptr || _fsExtra->format(); #endif - return primary_success && secondary_success; + const bool success = primary_success && secondary_success; +#if defined(NRF52_PLATFORM) + // A successful explicit erase creates a new empty authoritative store. A + // failed erase must keep the boot's load quarantine latched. + if (success) { + _contact_load_incomplete = false; + _identity_creation_blocked = false; + _prefs_load_incomplete = false; + } +#endif + return success; #elif defined(RP2040_PLATFORM) return LittleFS.format(); #elif defined(ESP32) @@ -370,129 +423,244 @@ bool DataStore::repairInternalExtraFS() { } bool DataStore::loadMainIdentity(mesh::LocalIdentity &identity) { +#if defined(NRF52_PLATFORM) + if (_primary_storage_unavailable) return false; + + bool identity_exists = false; + if (!contactPathPresence(_fs, "/_main.id", identity_exists)) { + _identity_creation_blocked = true; + return false; + } + if (!identity_exists) return false; + + if (!identity_store.load("_main", identity)) { + // A present identity which cannot be read or decoded is not a fresh + // install. Never replace it with a newly generated key this boot. + _identity_creation_blocked = true; + return false; + } + // A verified primary identity is canonical even when only contact/channel + // authority on an unavailable secondary remains unknown. + _identity_creation_blocked = false; + return true; +#else return identity_store.load("_main", identity); +#endif +} + +bool DataStore::canCreateMainIdentity() const { +#if defined(NRF52_PLATFORM) + return !_identity_creation_blocked; +#else + return true; +#endif } bool DataStore::saveMainIdentity(const mesh::LocalIdentity &identity) { +#if defined(NRF52_PLATFORM) + if (_identity_creation_blocked) return false; +#endif return identity_store.save("_main", identity); } -void DataStore::loadPrefs(CompanionNodePrefs& prefs, double& node_lat, double& node_lon) { - if (_fs->exists("/new_prefs")) { - loadPrefsInt("/new_prefs", prefs, node_lat, node_lon); // new filename - } else if (_fs->exists("/node_prefs")) { - loadPrefsInt("/node_prefs", prefs, node_lat, node_lon); - if (savePrefs(prefs, node_lat, node_lon)) { - _fs->remove("/node_prefs"); // remove old only after verified replacement - } +bool DataStore::loadPrefs(CompanionNodePrefs& prefs, double& node_lat, + double& node_lon) { +#if defined(NRF52_PLATFORM) + if (_primary_storage_unavailable + || (_prefs_load_incomplete && !_secondary_authority_unknown)) { + _prefs_load_incomplete = true; + return false; } + + bool new_prefs_exists = false; + if (!contactPathPresence(_fs, "/new_prefs", new_prefs_exists)) { + _prefs_load_incomplete = true; + return false; + } + if (new_prefs_exists) { + const bool loaded = loadPrefsInt( + "/new_prefs", prefs, node_lat, node_lon); + _prefs_load_incomplete = !loaded; + return loaded; + } + + bool legacy_prefs_exists = false; + if (!contactPathPresence(_fs, "/node_prefs", legacy_prefs_exists)) { + _prefs_load_incomplete = true; + return false; + } + if (!legacy_prefs_exists) { + if (_secondary_authority_unknown) { + _prefs_load_incomplete = true; + return false; + } + return true; + } +#else + if (_fs->exists("/new_prefs")) { + return loadPrefsInt("/new_prefs", prefs, node_lat, node_lon); + } + if (!_fs->exists("/node_prefs")) return true; +#endif + + if (!loadPrefsInt("/node_prefs", prefs, node_lat, node_lon)) { +#if defined(NRF52_PLATFORM) + _prefs_load_incomplete = true; +#endif + return false; + } +#if defined(NRF52_PLATFORM) + // A verified primary legacy image is authoritative even if an unavailable + // secondary might contain a duplicate. Permit its atomic in-place upgrade. + _prefs_load_incomplete = false; +#endif + if (savePrefs(prefs, node_lat, node_lon)) { + _fs->remove("/node_prefs"); // remove old only after verified replacement + } + return true; } -void DataStore::loadPrefsInt(const char *filename, CompanionNodePrefs& _prefs, double& node_lat, double& node_lon) { +bool DataStore::loadPrefsInt(const char *filename, + CompanionNodePrefs& _prefs, double& node_lat, + double& node_lon) { File file = openRead(_fs, filename); if (file) { - uint8_t pad[8]; - - file.read((uint8_t *)&_prefs.airtime_factor, sizeof(float)); // 0 - file.read((uint8_t *)_prefs.node_name, sizeof(_prefs.node_name)); // 4 - file.read(pad, 4); // 36 - file.read((uint8_t *)&node_lat, sizeof(node_lat)); // 40 - file.read((uint8_t *)&node_lon, sizeof(node_lon)); // 48 - file.read((uint8_t *)&_prefs.freq, sizeof(_prefs.freq)); // 56 - file.read((uint8_t *)&_prefs.sf, sizeof(_prefs.sf)); // 60 - file.read((uint8_t *)&_prefs.cr, sizeof(_prefs.cr)); // 61 - file.read((uint8_t *)&_prefs.client_repeat, sizeof(_prefs.client_repeat)); // 62 - file.read((uint8_t *)&_prefs.manual_add_contacts, sizeof(_prefs.manual_add_contacts)); // 63 - file.read((uint8_t *)&_prefs.bw, sizeof(_prefs.bw)); // 64 - file.read((uint8_t *)&_prefs.tx_power_dbm, sizeof(_prefs.tx_power_dbm)); // 68 - file.read((uint8_t *)&_prefs.telemetry_mode_base, sizeof(_prefs.telemetry_mode_base)); // 69 - file.read((uint8_t *)&_prefs.telemetry_mode_loc, sizeof(_prefs.telemetry_mode_loc)); // 70 - file.read((uint8_t *)&_prefs.telemetry_mode_env, sizeof(_prefs.telemetry_mode_env)); // 71 - file.read((uint8_t *)&_prefs.rx_delay_base, sizeof(_prefs.rx_delay_base)); // 72 - file.read((uint8_t *)&_prefs.advert_loc_policy, sizeof(_prefs.advert_loc_policy)); // 76 - file.read((uint8_t *)&_prefs.multi_acks, sizeof(_prefs.multi_acks)); // 77 - file.read((uint8_t *)&_prefs.path_hash_mode, sizeof(_prefs.path_hash_mode)); // 78 - file.read(pad, 1); // 79 - file.read((uint8_t *)&_prefs.ble_pin, sizeof(_prefs.ble_pin)); // 80 - file.read((uint8_t *)&_prefs.buzzer_quiet, sizeof(_prefs.buzzer_quiet)); // 84 - file.read((uint8_t *)&_prefs.gps_enabled, sizeof(_prefs.gps_enabled)); // 85 - file.read((uint8_t *)&_prefs.gps_interval, sizeof(_prefs.gps_interval)); // 86 - file.read((uint8_t *)&_prefs.autoadd_config, sizeof(_prefs.autoadd_config)); // 87 - file.read((uint8_t *)&_prefs.autoadd_max_hops, sizeof(_prefs.autoadd_max_hops)); // 88 - file.read((uint8_t *)&_prefs.rx_boosted_gain, sizeof(_prefs.rx_boosted_gain)); // 89 - file.read((uint8_t *)_prefs.default_scope_name, sizeof(_prefs.default_scope_name)); // 90 - file.read((uint8_t *)_prefs.default_scope_key, sizeof(_prefs.default_scope_key)); // 121 - file.read((uint8_t *)&_prefs.radio_fem_rxgain, sizeof(_prefs.radio_fem_rxgain)); // 122 - file.read((uint8_t *)&_prefs.radio_fem_rxgain_override, - sizeof(_prefs.radio_fem_rxgain_override)); // 123 - if (file.available() >= (int)sizeof(_prefs.vibe_quiet)) { - file.read((uint8_t *)&_prefs.vibe_quiet, sizeof(_prefs.vibe_quiet)); // 124 - } - if (file.available() >= (int)sizeof(_prefs.radio_fem_txgain)) { - file.read((uint8_t *)&_prefs.radio_fem_txgain, sizeof(_prefs.radio_fem_txgain)); // 125 - } - const size_t rxps_tail_size = sizeof(_prefs.rx_powersaving_enabled) - + sizeof(_prefs.rx_ps_rx_us) + sizeof(_prefs.rx_ps_sleep_us) - + sizeof(_prefs.rx_ps_level) + sizeof(_prefs.rx_ps_preamble); - if (file.available() >= (int)rxps_tail_size) { - file.read((uint8_t *)&_prefs.rx_powersaving_enabled, - sizeof(_prefs.rx_powersaving_enabled)); // 126 - file.read((uint8_t *)&_prefs.rx_ps_rx_us, sizeof(_prefs.rx_ps_rx_us)); // 127 - file.read((uint8_t *)&_prefs.rx_ps_sleep_us, sizeof(_prefs.rx_ps_sleep_us)); // 131 - file.read((uint8_t *)&_prefs.rx_ps_level, sizeof(_prefs.rx_ps_level)); // 135 - file.read((uint8_t *)&_prefs.rx_ps_preamble, sizeof(_prefs.rx_ps_preamble)); // 136 - if (file.available() >= (int)sizeof(_prefs.powersaving_enabled)) { - file.read((uint8_t *)&_prefs.powersaving_enabled, - sizeof(_prefs.powersaving_enabled)); // 137 - if (file.available() >= (int)sizeof(_prefs.wifi_enabled)) { - file.read((uint8_t *)&_prefs.wifi_enabled, - sizeof(_prefs.wifi_enabled)); // 138 - if (file.available() >= (int)sizeof(_prefs.powersaving_policy_version)) { - file.read((uint8_t *)&_prefs.powersaving_policy_version, - sizeof(_prefs.powersaving_policy_version)); // 139 - if (file.available() >= (int)sizeof(_prefs.usb_logging_enabled)) { - file.read((uint8_t *)&_prefs.usb_logging_enabled, - sizeof(_prefs.usb_logging_enabled)); // 140 - if (file.available() >= (int)sizeof(_prefs.bluetooth_name)) { - file.read((uint8_t *)_prefs.bluetooth_name, - sizeof(_prefs.bluetooth_name)); // 141 - if (file.available() - >= (int)sizeof(_prefs.display_rotation_degrees)) { - file.read((uint8_t *)&_prefs.display_rotation_degrees, - sizeof(_prefs.display_rotation_degrees)); - if (file.available() >= (int)sizeof(_prefs.cad_enabled)) { - file.read((uint8_t *)&_prefs.cad_enabled, - sizeof(_prefs.cad_enabled)); - if (file.available() - >= (int)sizeof(_prefs.cad_scan_timeout_ms)) { - file.read((uint8_t *)&_prefs.cad_scan_timeout_ms, - sizeof(_prefs.cad_scan_timeout_ms)); - if (file.available() - >= (int)sizeof(_prefs.cad_retry_delay_ms)) { - file.read((uint8_t *)&_prefs.cad_retry_delay_ms, - sizeof(_prefs.cad_retry_delay_ms)); - if (file.available() - >= (int)sizeof(_prefs.cad_max_duration_ms)) { - file.read((uint8_t *)&_prefs.cad_max_duration_ms, - sizeof(_prefs.cad_max_duration_ms)); - } - } - } - } - } - } - } - } - } + CompanionNodePrefs loaded_prefs = _prefs; + double loaded_lat = node_lat; + double loaded_lon = node_lon; + // The original image ended after ble_pin at byte 84. Later releases only + // appended fields, sometimes as an indivisible group. Accept every format + // actually emitted by those releases, but reject a truncated field/group + // or an unknown tail before any value reaches the live preferences. + static const uint32_t MIN_PREFS_SIZE = 84; + static const uint32_t KNOWN_PREFS_SIZES[] = { + 84, 85, 90, 91, 92, 93, 140, 141, 142, 143, 144, 155, + 156, 157, 158, 159, + 159 + sizeof(loaded_prefs.bluetooth_name), + 161 + sizeof(loaded_prefs.bluetooth_name), + 168 + sizeof(loaded_prefs.bluetooth_name), + }; + const uint32_t prefs_size = file.size(); + bool known_size = false; + for (size_t i = 0; + i < sizeof(KNOWN_PREFS_SIZES) / sizeof(KNOWN_PREFS_SIZES[0]); i++) { + if (prefs_size == KNOWN_PREFS_SIZES[i]) { + known_size = true; + break; } } + if (prefs_size < MIN_PREFS_SIZE || !known_size) { + file.close(); + return false; + } + bool success = true; + auto readField = [&file, &success](void* dest, size_t size) -> bool { + if (!success + || file.read(static_cast(dest), size) != size) { + success = false; + return false; + } + return true; + }; + auto readOptionalField = [&file, &readField](void* dest, + size_t size) -> bool { + return file.available() == 0 || readField(dest, size); + }; + uint8_t pad[8]; + readField(&loaded_prefs.airtime_factor, sizeof(float)); // 0 + readField(loaded_prefs.node_name, sizeof(loaded_prefs.node_name)); // 4 + readField(pad, 4); // 36 + readField(&loaded_lat, sizeof(loaded_lat)); // 40 + readField(&loaded_lon, sizeof(loaded_lon)); // 48 + readField(&loaded_prefs.freq, sizeof(loaded_prefs.freq)); // 56 + readField(&loaded_prefs.sf, sizeof(loaded_prefs.sf)); // 60 + readField(&loaded_prefs.cr, sizeof(loaded_prefs.cr)); // 61 + readField(&loaded_prefs.client_repeat, sizeof(loaded_prefs.client_repeat)); // 62 + readField(&loaded_prefs.manual_add_contacts, sizeof(loaded_prefs.manual_add_contacts)); // 63 + readField(&loaded_prefs.bw, sizeof(loaded_prefs.bw)); // 64 + readField(&loaded_prefs.tx_power_dbm, sizeof(loaded_prefs.tx_power_dbm)); // 68 + readField(&loaded_prefs.telemetry_mode_base, sizeof(loaded_prefs.telemetry_mode_base)); // 69 + readField(&loaded_prefs.telemetry_mode_loc, sizeof(loaded_prefs.telemetry_mode_loc)); // 70 + readField(&loaded_prefs.telemetry_mode_env, sizeof(loaded_prefs.telemetry_mode_env)); // 71 + readField(&loaded_prefs.rx_delay_base, sizeof(loaded_prefs.rx_delay_base)); // 72 + readField(&loaded_prefs.advert_loc_policy, sizeof(loaded_prefs.advert_loc_policy)); // 76 + readField(&loaded_prefs.multi_acks, sizeof(loaded_prefs.multi_acks)); // 77 + readField(&loaded_prefs.path_hash_mode, sizeof(loaded_prefs.path_hash_mode)); // 78 + readField(pad, 1); // 79 + readField(&loaded_prefs.ble_pin, sizeof(loaded_prefs.ble_pin)); // 80 + readOptionalField(&loaded_prefs.buzzer_quiet, + sizeof(loaded_prefs.buzzer_quiet)); // 84 + readOptionalField(&loaded_prefs.gps_enabled, + sizeof(loaded_prefs.gps_enabled)); // 85 + readOptionalField(&loaded_prefs.gps_interval, + sizeof(loaded_prefs.gps_interval)); // 86 + readOptionalField(&loaded_prefs.autoadd_config, + sizeof(loaded_prefs.autoadd_config)); // 90 + readOptionalField(&loaded_prefs.autoadd_max_hops, + sizeof(loaded_prefs.autoadd_max_hops)); // 91 + readOptionalField(&loaded_prefs.rx_boosted_gain, + sizeof(loaded_prefs.rx_boosted_gain)); // 92 + readOptionalField(loaded_prefs.default_scope_name, + sizeof(loaded_prefs.default_scope_name)); // 93 + readOptionalField(loaded_prefs.default_scope_key, + sizeof(loaded_prefs.default_scope_key)); // 124 + readOptionalField(&loaded_prefs.radio_fem_rxgain, + sizeof(loaded_prefs.radio_fem_rxgain)); // 140 + readOptionalField(&loaded_prefs.radio_fem_rxgain_override, + sizeof(loaded_prefs.radio_fem_rxgain_override)); // 141 + readOptionalField(&loaded_prefs.vibe_quiet, + sizeof(loaded_prefs.vibe_quiet)); // 142 + readOptionalField(&loaded_prefs.radio_fem_txgain, + sizeof(loaded_prefs.radio_fem_txgain)); // 143 + readOptionalField(&loaded_prefs.rx_powersaving_enabled, + sizeof(loaded_prefs.rx_powersaving_enabled)); // 144 + readOptionalField(&loaded_prefs.rx_ps_rx_us, + sizeof(loaded_prefs.rx_ps_rx_us)); // 145 + readOptionalField(&loaded_prefs.rx_ps_sleep_us, + sizeof(loaded_prefs.rx_ps_sleep_us)); // 149 + readOptionalField(&loaded_prefs.rx_ps_level, + sizeof(loaded_prefs.rx_ps_level)); // 153 + readOptionalField(&loaded_prefs.rx_ps_preamble, + sizeof(loaded_prefs.rx_ps_preamble)); // 154 + readOptionalField(&loaded_prefs.powersaving_enabled, + sizeof(loaded_prefs.powersaving_enabled)); // 155 + readOptionalField(&loaded_prefs.wifi_enabled, + sizeof(loaded_prefs.wifi_enabled)); // 156 + readOptionalField(&loaded_prefs.powersaving_policy_version, + sizeof(loaded_prefs.powersaving_policy_version)); // 157 + readOptionalField(&loaded_prefs.usb_logging_enabled, + sizeof(loaded_prefs.usb_logging_enabled)); // 158 + readOptionalField(loaded_prefs.bluetooth_name, + sizeof(loaded_prefs.bluetooth_name)); // 159 + readOptionalField(&loaded_prefs.display_rotation_degrees, + sizeof(loaded_prefs.display_rotation_degrees)); // 191 + readOptionalField(&loaded_prefs.cad_enabled, + sizeof(loaded_prefs.cad_enabled)); // 193 + readOptionalField(&loaded_prefs.cad_scan_timeout_ms, + sizeof(loaded_prefs.cad_scan_timeout_ms)); // 194 + readOptionalField(&loaded_prefs.cad_retry_delay_ms, + sizeof(loaded_prefs.cad_retry_delay_ms)); // 196 + readOptionalField(&loaded_prefs.cad_max_duration_ms, + sizeof(loaded_prefs.cad_max_duration_ms)); // 198 + + // Any bytes left over form only part of a historically appended field. + // Preserve the file and defaults rather than treating that tail as EOF. + success = success && file.available() == 0; file.close(); + if (!success) return false; + _prefs = loaded_prefs; + node_lat = loaded_lat; + node_lon = loaded_lon; + return true; } + return false; } bool DataStore::savePrefs(const CompanionNodePrefs& _prefs, double node_lat, double node_lon) { #if defined(NRF52_PLATFORM) + if (_prefs_load_incomplete) return false; + if (_primary_storage_unavailable) return false; mesh::AtomicFileWriter file(_fs, "/new_prefs"); #else File file = openWrite(_fs, "/new_prefs"); @@ -632,6 +800,39 @@ static const char* CONTACT_MIGRATION_MARKER = "/contacts4.mig"; static const char* SECONDARY_MIGRATION_JOURNAL = "/.extrafs.mig"; static const uint8_t SECONDARY_MIGRATION_PENDING = 1; static const uint8_t SECONDARY_MIGRATION_COMMITTED = 2; +#if defined(MESHCORE_EXTRAFS_HIL) +static const char* HIL_CONTACT_PAGE_FAILURE_MARKER = "/__hil.readfail"; +static const uint8_t HIL_CONTACT_STAT_FAILURE_FLAG = 0x80; +#endif + +static mesh::storage::ContactPathState statContactPath( + FILESYSTEM* fs, const char* path, uint32_t* size) { + if (fs == nullptr || path == nullptr) { + return mesh::storage::ContactPathState::IO_ERROR; + } + + struct lfs_info info; + fs->_lockFS(); + const int result = lfs_stat(fs->_getFS(), path, &info); + fs->_unlockFS(); + const mesh::storage::ContactPathState state = + mesh::storage::classifyContactPathStat(result, LFS_ERR_NOENT); + if (state == mesh::storage::ContactPathState::PRESENT && size != nullptr) { + *size = info.size; + } + if (state == mesh::storage::ContactPathState::IO_ERROR) { + MESH_DEBUG_PRINTLN( + "DataStore: storage path stat failed for %s: %d", path, result); + } + return state; +} + +static bool contactPathPresence(FILESYSTEM* fs, const char* path, + bool& present, uint32_t* size) { + const mesh::storage::ContactPathState state = statContactPath(fs, path, size); + present = state == mesh::storage::ContactPathState::PRESENT; + return state != mesh::storage::ContactPathState::IO_ERROR; +} static void makeContactPagePath(uint8_t page, char path[24]) { snprintf(path, 24, "/contacts4_%02u", (unsigned)page); @@ -648,9 +849,11 @@ static void discardInvalidContactPage(FILESYSTEM* fs, const char* path, } } -void DataStore::resetContactPageState() { +void DataStore::resetContactPageState(bool clear_incomplete) { _contact_slots.clear(); _dirty_contact_pages.clearAll(); + _unread_contact_pages.clearAll(); + if (clear_incomplete) _contact_load_incomplete = false; memset(_contact_page_generations, 0, sizeof(_contact_page_generations)); _legacy_contacts_pending_cleanup = false; _legacy_migration_ready = false; @@ -659,7 +862,11 @@ void DataStore::resetContactPageState() { bool DataStore::prepareLegacyContactMigration() { FILESYSTEM* fs = _getContactsChannelsFS(); - if (fs->exists(CONTACT_MIGRATION_MARKER)) { + bool marker_exists = false; + if (!contactPathPresence(fs, CONTACT_MIGRATION_MARKER, marker_exists)) { + return false; + } + if (marker_exists) { _legacy_migration_ready = true; return true; } @@ -671,11 +878,15 @@ bool DataStore::prepareLegacyContactMigration() { for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { char path[24]; makeContactPagePath(page, path); - if (fs->exists(path) && !fs->remove(path)) clean = false; + bool path_exists = false; + if (!contactPathPresence(fs, path, path_exists)) return false; + if (path_exists && !fs->remove(path)) clean = false; char temp_path[28]; snprintf(temp_path, sizeof(temp_path), "%s.tmp", path); - if (fs->exists(temp_path) && !fs->remove(temp_path)) clean = false; + bool temp_exists = false; + if (!contactPathPresence(fs, temp_path, temp_exists)) return false; + if (temp_exists && !fs->remove(temp_path)) clean = false; } if (!clean) { MESH_DEBUG_PRINTLN("DataStore: could not clear stale contact pages before migration"); @@ -690,20 +901,61 @@ bool DataStore::prepareLegacyContactMigration() { return _legacy_migration_ready; } -bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot) { +bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot, + uint32_t expected_page_mask) { bool any_page_file = false; FILESYSTEM* fs = _getContactsChannelsFS(); + auto quarantineUnreadPage = [this](uint8_t page) { + // Fail closed if an otherwise present page cannot be read. Reserving its + // slots prevents a new contact from overwriting records which a later boot + // may recover successfully. + for (uint8_t index = 0; + index < mesh::storage::CONTACTS_PER_PAGE; index++) { + _contact_slots.reserve( + (uint16_t)page * mesh::storage::CONTACTS_PER_PAGE + index); + } + _unread_contact_pages.mark(page); + _contact_load_incomplete = true; + }; for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { + if ((expected_page_mask & (1UL << page)) == 0) continue; + char path[24]; makeContactPagePath(page, path); - if (!fs->exists(path)) continue; + // The first source-selection pass proved this page existed. Do not ask + // exists() a second time and silently reinterpret a transient failure as + // an empty slot range; opening the captured page either succeeds or puts + // that range into fail-closed quarantine. any_page_file = true; +#if defined(MESHCORE_EXTRAFS_HIL) + // Direct-attached HIL can request one fail-closed boot without damaging + // the page. Consume the marker before quarantine so the next reboot + // exercises normal recovery from the untouched source. + if (fs->exists(HIL_CONTACT_PAGE_FAILURE_MARKER)) { + File marker = openRead(fs, HIL_CONTACT_PAGE_FAILURE_MARKER); + const int fail_page = marker ? marker.read() : -1; + if (marker) marker.close(); + if (fail_page == page && fs->remove(HIL_CONTACT_PAGE_FAILURE_MARKER) + && !fs->exists(HIL_CONTACT_PAGE_FAILURE_MARKER)) { + MESH_DEBUG_PRINTLN( + "DataStore: HIL quarantining contact page %u", page); + quarantineUnreadPage(page); + continue; + } + } +#endif + File file = openRead(fs, path); - if (!file || file.size() != mesh::storage::CONTACT_PAGE_FILE_SIZE) { + if (!file) { + MESH_DEBUG_PRINTLN("DataStore: contact page %u could not be opened", page); + quarantineUnreadPage(page); + continue; + } + if (file.size() != mesh::storage::CONTACT_PAGE_FILE_SIZE) { MESH_DEBUG_PRINTLN("DataStore: ignoring invalid contact page %u", page); - if (file) file.close(); + file.close(); discardInvalidContactPage(fs, path, page); _dirty_contact_pages.mark(page); continue; @@ -711,23 +963,20 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot) { uint8_t raw_header[mesh::storage::CONTACT_PAGE_HEADER_SIZE]; mesh::storage::ContactPageHeader header; - bool valid = file.read(raw_header, sizeof(raw_header)) == sizeof(raw_header) + const bool header_read = + file.read(raw_header, sizeof(raw_header)) == sizeof(raw_header); + const bool header_valid = header_read && mesh::storage::decodeContactPageHeader(raw_header, page, header); - - uint32_t crc = 0xFFFFFFFFUL; - uint16_t remaining = mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE; - uint8_t chunk[64]; - while (valid && remaining > 0) { - const uint16_t count = remaining < sizeof(chunk) ? remaining : sizeof(chunk); - if (file.read(chunk, count) != count) { - valid = false; - break; - } - crc = mesh::storage::updateCRC32(crc, chunk, count); - remaining -= count; + if (!header_read) { + // A short read is not evidence that the atomically committed page is + // corrupt. Leave it untouched so a later boot can retry it. + MESH_DEBUG_PRINTLN( + "DataStore: contact page %u header could not be read", page); + file.close(); + quarantineUnreadPage(page); + continue; } - - if (!valid || crc != header.payload_crc) { + if (!header_valid) { MESH_DEBUG_PRINTLN("DataStore: contact page %u failed CRC/format validation", page); file.close(); discardInvalidContactPage(fs, path, page); @@ -735,17 +984,67 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot) { continue; } + // Keep a verified page snapshot in heap memory while constructing the RAM + // table. A second series of filesystem seeks could fail after only part of + // the page had been loaded; later rewriting that partial RAM view would + // permanently delete otherwise valid contacts. + uint8_t* payload = + (uint8_t*)malloc(mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE); + if (payload == nullptr) { + MESH_DEBUG_PRINTLN( + "DataStore: no memory to load contact page %u", page); + file.close(); + quarantineUnreadPage(page); + continue; + } + const bool payload_read = + file.read(payload, mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE) + == mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE; + file.close(); + if (!payload_read) { + MESH_DEBUG_PRINTLN( + "DataStore: contact page %u payload could not be read", page); + free(payload); + quarantineUnreadPage(page); + continue; + } + const uint32_t crc = mesh::storage::updateCRC32( + 0xFFFFFFFFUL, payload, mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE); + if (crc != header.payload_crc) { + MESH_DEBUG_PRINTLN("DataStore: contact page %u failed CRC/format validation", page); + free(payload); + discardInvalidContactPage(fs, path, page); + _dirty_contact_pages.mark(page); + continue; + } + + if ((header.occupied & ~mesh::storage::contactPageValidSlotMask()) != 0) { + MESH_DEBUG_PRINTLN( + "DataStore: repairing contact page %u high occupancy bits", page); + _dirty_contact_pages.mark(page); + } + _contact_page_generations[page] = header.generation; for (uint8_t index = 0; index < mesh::storage::CONTACTS_PER_PAGE; index++) { - if ((header.occupied & (1UL << index)) == 0) continue; - const uint16_t slot = (uint16_t)page * mesh::storage::CONTACTS_PER_PAGE + index; - if (!mesh::storage::loadSlotFromMigratedPage(slot, minimum_slot)) continue; - uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; - if (!file.seek(mesh::storage::CONTACT_PAGE_HEADER_SIZE - + (uint32_t)index * mesh::storage::CONTACT_RECORD_SIZE) - || file.read(record, sizeof(record)) != sizeof(record)) { - MESH_DEBUG_PRINTLN("DataStore: contact page %u slot %u could not be read", page, index); + const uint8_t* record = + &payload[(uint16_t)index * mesh::storage::CONTACT_RECORD_SIZE]; + + // The payload CRC covers every record, so occupancy can be reconstructed + // without trusting the separate header bitmask. This recovers a contact + // hidden by a flipped mask bit and ignores an empty slot exposed by one; + // the dirty page is later rewritten with the repaired mask. + const bool has_data = mesh::storage::contactRecordHasData(record); + const bool marked_occupied = + (header.occupied & (1UL << index)) != 0; + if (has_data != marked_occupied) { + MESH_DEBUG_PRINTLN( + "DataStore: repairing contact page %u occupancy slot %u", + page, index); + _dirty_contact_pages.mark(page); + } + if (!has_data + || !mesh::storage::loadSlotFromMigratedPage(slot, minimum_slot)) { continue; } @@ -758,11 +1057,25 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot) { contact.storage_slot = slot; if (!host->onContactLoaded(contact)) { _contact_slots.release(slot); - file.close(); + // A host capacity refusal is not a filesystem read error, but the RAM + // table is still not a complete representation of the durable page. + // Preserve the page and veto every mutation/rewrite until reboot. + MESH_DEBUG_PRINTLN( + "DataStore: contact host refused page %u slot %u; load incomplete", + page, index); + _contact_load_incomplete = true; + _dirty_contact_pages.clearAll(); + free(payload); return true; } } - file.close(); + free(payload); + } + if (hasIncompleteContactLoad()) { + // Occupancy/header repairs discovered elsewhere in this pass must wait for + // a clean reboot too. Rewriting any page from an incomplete contact table + // can turn a transient read failure into durable cross-page data loss. + _dirty_contact_pages.clearAll(); } return any_page_file; } @@ -838,28 +1151,96 @@ void DataStore::loadContacts(DataStoreHost* host) { // slot ownership from disk so stale pointers/slots from the previous in-RAM // contact table cannot collide with the reload. resetContactPageState(); + if (_contact_load_incomplete) { + // A failed load is latched for this boot. loadContacts() is also called by + // identity import, but only a reboot may retry storage and re-enable + // migration/mutation; otherwise an unrelated command could clear the veto. + MESH_DEBUG_PRINTLN( + "DataStore: contact load remains quarantined until reboot"); + return; + } FILESYSTEM* contacts_fs = _getContactsChannelsFS(); - bool any_page_file = false; +#if defined(MESHCORE_EXTRAFS_HIL) + // Reuse the read-failure marker's high bit for a one-boot source-discovery + // stat failure. Consume and verify the marker before injecting the error so + // the untouched page is recoverable on the very next reboot. Low-bit values + // remain armed for loadContactPages(), preserving the existing read hook. + int hil_stat_failure_page = -1; + bool hil_failure_marker_exists = false; + if (!contactPathPresence(contacts_fs, HIL_CONTACT_PAGE_FAILURE_MARKER, + hil_failure_marker_exists)) { + _contact_load_incomplete = true; + return; + } + if (hil_failure_marker_exists) { + File marker = openRead(contacts_fs, HIL_CONTACT_PAGE_FAILURE_MARKER); + const int encoded_failure = marker && marker.size() == 1 + ? marker.read() : -1; + if (marker) marker.close(); + if (encoded_failure >= 0 + && (encoded_failure & HIL_CONTACT_STAT_FAILURE_FLAG) != 0) { + const uint8_t page = static_cast( + encoded_failure & ~HIL_CONTACT_STAT_FAILURE_FLAG); + bool marker_remains = true; + const bool removed = contacts_fs->remove( + HIL_CONTACT_PAGE_FAILURE_MARKER); + const bool verified = contactPathPresence( + contacts_fs, HIL_CONTACT_PAGE_FAILURE_MARKER, marker_remains); + if (page >= mesh::storage::CONTACT_PAGE_COUNT || !removed || !verified + || marker_remains) { + MESH_DEBUG_PRINTLN( + "DataStore: HIL stat-failure marker could not be consumed safely"); + _contact_load_incomplete = true; + return; + } + hil_stat_failure_page = page; + } + } +#endif + uint32_t contact_page_presence = 0; for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { char path[24]; makeContactPagePath(page, path); - if (contacts_fs->exists(path)) { - any_page_file = true; + bool page_exists = false; +#if defined(MESHCORE_EXTRAFS_HIL) + if (page == hil_stat_failure_page) { + MESH_DEBUG_PRINTLN( + "DataStore: HIL injecting contact page %u stat failure", page); + _contact_load_incomplete = true; + return; } +#endif + if (!contactPathPresence(contacts_fs, path, page_exists)) { + _contact_load_incomplete = true; + return; + } + if (page_exists) { + contact_page_presence |= 1UL << page; + } + } + bool legacy_exists = false; + bool migration_marker_exists = false; + if (!contactPathPresence(contacts_fs, "/contacts3", legacy_exists) + || !contactPathPresence(contacts_fs, CONTACT_MIGRATION_MARKER, + migration_marker_exists)) { + _contact_load_incomplete = true; + return; } const mesh::storage::ContactStoreSource source = mesh::storage::chooseContactStoreSource( - contacts_fs->exists("/contacts3"), any_page_file); + legacy_exists, contact_page_presence != 0); if (source == mesh::storage::ContactStoreSource::PAGED) { + loadContactPages(host, 0, contact_page_presence); // A reset after the final legacy removal may leave this harmless marker. - if (contacts_fs->exists(CONTACT_MIGRATION_MARKER)) { + // Retire it only after every page captured above loaded successfully; an + // unread page must leave all recovery metadata untouched for reboot retry. + if (!hasIncompleteContactLoad() && migration_marker_exists) { contacts_fs->remove(CONTACT_MIGRATION_MARKER); } - loadContactPages(host, 0); return; } if (source == mesh::storage::ContactStoreSource::EMPTY) { - if (contacts_fs->exists(CONTACT_MIGRATION_MARKER)) { + if (migration_marker_exists) { contacts_fs->remove(CONTACT_MIGRATION_MARKER); } return; @@ -870,27 +1251,38 @@ void DataStore::loadContacts(DataStoreHost* host) { // then the corresponding legacy tail is truncated. On a reset between // those operations the still-present legacy prefix wins overlapping slots. _legacy_contacts_pending_cleanup = true; - _legacy_migration_ready = contacts_fs->exists(CONTACT_MIGRATION_MARKER); + _legacy_migration_ready = migration_marker_exists; #endif File file = openRead(_getContactsChannelsFS(), "/contacts3"); #if defined(NRF52_PLATFORM) if (!file) { - // Never delete or truncate a legacy source that could not be opened. + // Never delete, truncate, or write alongside a legacy source that could + // not be opened. A later reboot can retry the untouched authoritative + // file, but this boot must not accept a mutation into a competing page. MESH_DEBUG_PRINTLN("DataStore: legacy contacts exist but could not be read"); - _legacy_contacts_pending_cleanup = false; + _contact_load_incomplete = true; return; } - if (!mesh::storage::trustMigratedContactPages( - true, _legacy_migration_ready)) { - prepareLegacyContactMigration(); + + const size_t legacy_size = file.size(); + if (!mesh::storage::isValidLegacyContactFileSize(legacy_size)) { + MESH_DEBUG_PRINTLN( + "DataStore: legacy contacts size %lu is invalid; preserving source", + (unsigned long)legacy_size); + file.close(); + _contact_load_incomplete = true; + return; } #endif if (file) { bool full = false; #if defined(NRF52_PLATFORM) - _legacy_contact_count = mesh::storage::legacyContactCountForSize(file.size()); + _legacy_contact_count = + mesh::storage::legacyContactCountForSize(legacy_size); uint16_t record_index = 0; + bool legacy_read_failed = false; + bool legacy_host_refused = false; #endif while (!full #if defined(NRF52_PLATFORM) @@ -898,7 +1290,12 @@ void DataStore::loadContacts(DataStoreHost* host) { #endif ) { uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; - if (file.read(record, sizeof(record)) != sizeof(record)) break; + if (file.read(record, sizeof(record)) != sizeof(record)) { +#if defined(NRF52_PLATFORM) + legacy_read_failed = true; +#endif + break; + } ContactInfo contact; if (!deserializeContactRecord(record, contact)) { @@ -907,17 +1304,55 @@ void DataStore::loadContacts(DataStoreHost* host) { } #if defined(NRF52_PLATFORM) const uint16_t slot = record_index++; - if (!_contact_slots.reserve(slot)) break; + if (!_contact_slots.reserve(slot)) { + legacy_read_failed = true; + break; + } contact.storage_slot = slot; #endif if (!host->onContactLoaded(contact)) { full = true; #if defined(NRF52_PLATFORM) _contact_slots.release(slot); + legacy_host_refused = true; #endif } } file.close(); + +#if defined(NRF52_PLATFORM) + if (legacy_read_failed || legacy_host_refused + || record_index != _legacy_contact_count) { + if (legacy_host_refused) { + // Capacity refusal is semantically distinct from an I/O error, but the + // durable source still contains contacts absent from RAM. Preserve it + // and use the same mutation/migration veto until a clean load succeeds. + MESH_DEBUG_PRINTLN( + "DataStore: contact host capacity refused legacy record %u; load incomplete", + (unsigned)record_index); + } else { + MESH_DEBUG_PRINTLN( + "DataStore: short/incomplete legacy contact read at record %u of %u", + (unsigned)record_index, (unsigned)_legacy_contact_count); + } + _contact_load_incomplete = true; + return; + } + + // Only modify stale page artifacts after the complete authoritative legacy + // file has been loaded. A transient read failure above therefore leaves + // every possible recovery source untouched. + if (!mesh::storage::trustMigratedContactPages( + true, _legacy_migration_ready)) { + if (!prepareLegacyContactMigration()) { + _contact_load_incomplete = true; + return; + } + // prepareLegacyContactMigration() intentionally removed every stale page + // observed in the first pass before committing the marker. + contact_page_presence = 0; + } +#endif } #if defined(NRF52_PLATFORM) @@ -925,7 +1360,7 @@ void DataStore::loadContacts(DataStoreHost* host) { // Loading both sources this way resumes safely after every possible reset // point, including a reset after page rename but before legacy truncation. if (_legacy_migration_ready) { - loadContactPages(host, _legacy_contact_count); + loadContactPages(host, _legacy_contact_count, contact_page_presence); } #endif } @@ -978,6 +1413,7 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn bool DataStore::markContactDirty(const ContactInfo& contact) { #if defined(NRF52_PLATFORM) + if (hasIncompleteContactLoad()) return false; uint16_t slot = contact.storage_slot; if (!_contact_slots.isUsed(slot)) { slot = _contact_slots.allocate(); @@ -993,6 +1429,7 @@ bool DataStore::markContactDirty(const ContactInfo& contact) { bool DataStore::releaseContact(const ContactInfo& contact) { #if defined(NRF52_PLATFORM) + if (hasIncompleteContactLoad()) return false; const uint16_t slot = contact.storage_slot; if (!_contact_slots.release(slot)) return false; contact.storage_slot = mesh::storage::CONTACT_SLOT_NONE; @@ -1003,20 +1440,45 @@ bool DataStore::releaseContact(const ContactInfo& contact) { #endif } +bool DataStore::restoreContactSlot(const ContactInfo& contact, uint16_t slot) { +#if defined(NRF52_PLATFORM) + if (hasIncompleteContactLoad()) return false; + // Roll back a release into the exact slot it vacated. Allocating the first + // free slot here could move the record to another page; a reset between the + // two resulting page writes would then leave duplicate records on disk. + if (!_contact_slots.reserve(slot)) return false; + if (!_dirty_contact_pages.mark( + slot / mesh::storage::CONTACTS_PER_PAGE)) { + _contact_slots.release(slot); + return false; + } + contact.storage_slot = slot; + return true; +#else + (void)contact; + (void)slot; + return true; +#endif +} + #if defined(NRF52_PLATFORM) bool DataStore::truncateLegacyContacts(uint16_t remaining_contacts) { FILESYSTEM* fs = _getContactsChannelsFS(); if (remaining_contacts == 0) { - const bool removed = !fs->exists("/contacts3") || fs->remove("/contacts3"); - if (removed) { - if (fs->exists(CONTACT_MIGRATION_MARKER)) { - fs->remove(CONTACT_MIGRATION_MARKER); - } - _legacy_contact_count = 0; - _legacy_contacts_pending_cleanup = false; - _legacy_migration_ready = false; + bool legacy_exists = false; + if (!contactPathPresence(fs, "/contacts3", legacy_exists)) return false; + if (legacy_exists && !fs->remove("/contacts3")) return false; + + bool marker_exists = false; + if (!contactPathPresence(fs, CONTACT_MIGRATION_MARKER, marker_exists)) { + return false; } - return removed; + if (marker_exists && !fs->remove(CONTACT_MIGRATION_MARKER)) return false; + + _legacy_contact_count = 0; + _legacy_contacts_pending_cleanup = false; + _legacy_migration_ready = false; + return true; } File file = fs->open("/contacts3", FILE_O_WRITE); @@ -1044,12 +1506,22 @@ bool DataStore::truncateLegacyContacts(uint16_t remaining_contacts) { bool DataStore::serviceContactWrites(DataStoreHost* host, bool (*filter)(const ContactInfo& c)) { #if defined(NRF52_PLATFORM) + if (hasIncompleteContactLoad()) { + // Reboot is the recovery operation. Never migrate or rewrite from a RAM + // table which is known not to represent every durable contact. + return false; + } if (_legacy_contacts_pending_cleanup) { FILESYSTEM* fs = _getContactsChannelsFS(); - if (!fs->exists("/contacts3")) { - if (fs->exists(CONTACT_MIGRATION_MARKER)) { - fs->remove(CONTACT_MIGRATION_MARKER); + bool legacy_exists = false; + if (!contactPathPresence(fs, "/contacts3", legacy_exists)) return false; + if (!legacy_exists) { + bool marker_exists = false; + if (!contactPathPresence(fs, CONTACT_MIGRATION_MARKER, + marker_exists)) { + return false; } + if (marker_exists && !fs->remove(CONTACT_MIGRATION_MARKER)) return false; _legacy_contacts_pending_cleanup = false; _legacy_migration_ready = false; _legacy_contact_count = 0; @@ -1061,6 +1533,11 @@ bool DataStore::serviceContactWrites(DataStoreHost* host, } else { const uint8_t page = mesh::storage::legacyMigrationPage(_legacy_contact_count); + if ((_unread_contact_pages.bits() & (1UL << page)) != 0) { + // The legacy prefix cannot reconstruct post-prefix records from an + // unread committed page. Keep both sources untouched for reboot retry. + return false; + } if (page >= mesh::storage::CONTACT_PAGE_COUNT || !writeContactPage(host, page, filter)) { return false; @@ -1098,38 +1575,129 @@ bool DataStore::flushContactWrites(DataStoreHost* host, bool DataStore::hasPendingContactWrites() const { #if defined(NRF52_PLATFORM) - return !_dirty_contact_pages.empty() || _legacy_contacts_pending_cleanup; + return !hasIncompleteContactLoad() + && (!_dirty_contact_pages.empty() || _legacy_contacts_pending_cleanup); +#else + return false; +#endif +} + +bool DataStore::hasIncompleteContactLoad() const { +#if defined(NRF52_PLATFORM) + return _contact_load_incomplete || !_unread_contact_pages.empty(); #else return false; #endif } void DataStore::loadChannels(DataStoreHost* host) { - File file = openRead(_getContactsChannelsFS(), "/channels2"); - if (file) { - bool full = false; - uint8_t channel_idx = 0; - while (!full) { - ChannelDetails ch; - uint8_t unused[4]; +#if defined(NRF52_PLATFORM) + if (_contact_load_incomplete) { + MESH_DEBUG_PRINTLN( + "DataStore: channel load quarantined after storage I/O failure"); + return; + } - bool success = (file.read(unused, 4) == 4); - success = success && (file.read((uint8_t *)ch.name, 32) == 32); - success = success && (file.read((uint8_t *)ch.channel.secret, 32) == 32); + FILESYSTEM* contacts_fs = _getContactsChannelsFS(); + bool channels_exist = false; + uint32_t channels_size = 0; + if (!contactPathPresence(contacts_fs, "/channels2", channels_exist, + &channels_size)) { + _contact_load_incomplete = true; + return; + } + if (!channels_exist) return; - if (!success) break; // EOF + static const uint32_t CHANNEL_RECORD_SIZE = 4 + 32 + 32; + if ((channels_size % CHANNEL_RECORD_SIZE) != 0 + || channels_size / CHANNEL_RECORD_SIZE > MAX_GROUP_CHANNELS) { + MESH_DEBUG_PRINTLN( + "DataStore: invalid channels file size; storage quarantined"); + _contact_load_incomplete = true; + return; + } +#endif + File file = openRead(_getContactsChannelsFS(), "/channels2"); +#if defined(NRF52_PLATFORM) + if (!file || file.size() != channels_size) { + if (file) file.close(); + MESH_DEBUG_PRINTLN( + "DataStore: channels file could not be opened consistently; storage quarantined"); + _contact_load_incomplete = true; + return; + } - if (host->onChannelLoaded(channel_idx, ch)) { - channel_idx++; - } else { - full = true; - } - } - file.close(); + const uint8_t channel_count = + static_cast(channels_size / CHANNEL_RECORD_SIZE); + ChannelDetails* loaded = channel_count == 0 ? nullptr + : static_cast( + malloc(sizeof(ChannelDetails) * channel_count)); + if (channel_count != 0 && loaded == nullptr) { + file.close(); + MESH_DEBUG_PRINTLN( + "DataStore: no memory for channels snapshot; storage quarantined"); + _contact_load_incomplete = true; + return; + } + bool success = true; + for (uint8_t channel_idx = 0; channel_idx < channel_count; channel_idx++) { + uint8_t unused[4]; + success = file.read(unused, sizeof(unused)) == sizeof(unused) + && file.read(reinterpret_cast(loaded[channel_idx].name), + sizeof(loaded[channel_idx].name)) + == sizeof(loaded[channel_idx].name) + && file.read(loaded[channel_idx].channel.secret, + sizeof(loaded[channel_idx].channel.secret)) + == sizeof(loaded[channel_idx].channel.secret); + if (!success) break; + } + file.close(); + if (!success) { + free(loaded); + MESH_DEBUG_PRINTLN( + "DataStore: channels file read failed; storage quarantined"); + _contact_load_incomplete = true; + return; + } + for (uint8_t channel_idx = 0; channel_idx < channel_count; channel_idx++) { + if (!host->onChannelLoaded(channel_idx, loaded[channel_idx])) { + free(loaded); + MESH_DEBUG_PRINTLN( + "DataStore: channel host refused durable record; storage quarantined"); + _contact_load_incomplete = true; + return; } + } + free(loaded); +#else + if (file) { + bool full = false; + uint8_t channel_idx = 0; + while (!full) { + ChannelDetails ch; + uint8_t unused[4]; + + bool success = (file.read(unused, 4) == 4); + success = success && (file.read((uint8_t *)ch.name, 32) == 32); + success = success && (file.read((uint8_t *)ch.channel.secret, 32) == 32); + + if (!success) break; // EOF + + if (host->onChannelLoaded(channel_idx, ch)) { + channel_idx++; + } else { + full = true; + } + } + file.close(); + } +#endif } bool DataStore::saveChannels(DataStoreHost* host) { +#if defined(NRF52_PLATFORM) + if (_contact_load_incomplete) return false; +#endif #if defined(NRF52_PLATFORM) mesh::AtomicFileWriter file(_getContactsChannelsFS(), "/channels2"); #else @@ -1260,14 +1828,42 @@ bool DataStore::migrateToSecondaryFS() { }; auto copy = [this, &filesEqual](FILESYSTEM* source_fs, FILESYSTEM* dest_fs, - const char* path) -> bool { + const char* path, + bool exact_snapshot) -> bool { +#if defined(NRF52_PLATFORM) + bool source_exists = false; + if (!contactPathPresence(source_fs, path, source_exists)) return false; + if (!source_exists) { + if (!exact_snapshot) return true; + + // A reset during a previous Pending attempt can leave a destination-only + // file. Primary is still authoritative until COMMITTED, so the retry + // must remove that stale file rather than blessing it into the snapshot. + bool destination_exists = false; + if (!contactPathPresence(dest_fs, path, destination_exists)) return false; + if (!destination_exists) return true; + if (!dest_fs->remove(path)) return false; + bool destination_remains = true; + return contactPathPresence(dest_fs, path, destination_remains) + && !destination_remains; + } + + bool destination_exists = false; + if (!contactPathPresence(dest_fs, path, destination_exists)) return false; +#else if (!source_fs->exists(path)) return true; - if (dest_fs->exists(path)) { + const bool destination_exists = dest_fs->exists(path); +#endif + if (destination_exists) { if (filesEqual(source_fs, dest_fs, path)) { return true; } - MESH_DEBUG_PRINTLN("DataStore: migration conflict for %s; preserving both copies", path); - return false; + if (!exact_snapshot) { + MESH_DEBUG_PRINTLN("DataStore: migration conflict for %s; preserving both copies", path); + return false; + } + // Pending migration destinations are non-authoritative. Fall through to + // AtomicFileWriter so this stale copy is replaced as one transaction. } File source = openRead(source_fs, path); @@ -1327,19 +1923,47 @@ bool DataStore::migrateToSecondaryFS() { return true; }; +#if !defined(NRF52_PLATFORM) auto move = [©](FILESYSTEM* source_fs, FILESYSTEM* dest_fs, const char* path) -> bool { if (!source_fs->exists(path)) return true; - return copy(source_fs, dest_fs, path) && source_fs->remove(path); + return copy(source_fs, dest_fs, path, false) && source_fs->remove(path); }; +#endif auto migratePrimarySources = [this, ©, &filesEqual]() -> bool { bool success = true; for (size_t i = 0; i < sizeof(to_primary) / sizeof(to_primary[0]); i++) { const char* path = to_primary[i]; - if (!_fsExtra->exists(path)) continue; +#if defined(NRF52_PLATFORM) + const bool is_identity = strcmp(path, "/_main.id") == 0; + const bool is_preferences = strcmp(path, "/new_prefs") == 0; +#endif + bool primary_exists = false; +#if defined(NRF52_PLATFORM) + if (!contactPathPresence(_fs, path, primary_exists)) { + if (is_identity) _identity_creation_blocked = true; + if (is_preferences) _prefs_load_incomplete = true; + return false; + } - if (_fs->exists(path)) { + bool secondary_exists = false; + if (!contactPathPresence(_fsExtra, path, secondary_exists)) { + // Identity and preferences on primary are canonical. If that copy is + // already present, a secondary metadata error only prevents optional + // duplicate cleanup and is irrelevant to their availability. + if (primary_exists) continue; + if (is_identity) _identity_creation_blocked = true; + if (is_preferences) _prefs_load_incomplete = true; + return false; + } +#else + primary_exists = _fs->exists(path); + const bool secondary_exists = _fsExtra->exists(path); +#endif + if (!secondary_exists) continue; + + if (primary_exists) { if (!filesEqual(_fsExtra, _fs, path)) { // Identity/preferences are canonical on primary. A differing legacy // secondary copy is useful forensic data, but it must not prevent @@ -1347,12 +1971,29 @@ bool DataStore::migrateToSecondaryFS() { MESH_DEBUG_PRINTLN("DataStore: preserving conflicting legacy secondary %s; primary remains authoritative", path); continue; } - } else if (!copy(_fsExtra, _fs, path)) { - // Without any primary copy MyMesh would generate a replacement - // identity or persist default preferences, so this is the only - // primary-source condition which must block the storage handoff. - success = false; - continue; + } else { + // The required source can disappear between its discovery probe and + // copy()'s own stat. Only a verified destination proves recovery. + bool recovered_primary = copy(_fsExtra, _fs, path, false); +#if defined(NRF52_PLATFORM) + bool recovered_primary_exists = false; + recovered_primary = recovered_primary + && contactPathPresence(_fs, path, recovered_primary_exists) + && recovered_primary_exists; +#else + recovered_primary = recovered_primary && _fs->exists(path); +#endif + if (!recovered_primary) { + // Without any primary copy MyMesh would generate a replacement + // identity or persist default preferences, so this is the only + // primary-source condition which must block the storage handoff. +#if defined(NRF52_PLATFORM) + if (is_identity) _identity_creation_blocked = true; + if (is_preferences) _prefs_load_incomplete = true; +#endif + success = false; + continue; + } } // At this point primary has an exact verified copy. Failure to retire a @@ -1365,27 +2006,22 @@ bool DataStore::migrateToSecondaryFS() { }; #if defined(NRF52_PLATFORM) - // Identity and preferences are always read from the primary filesystem. - // Recover legacy/test-layout copies before the contact/channel transaction - // can take an early fallback return; otherwise MyMesh may generate a new - // identity or persist default preferences while the real copy is stranded - // on the now-disabled secondary filesystem. - if (!migratePrimarySources()) { - MESH_DEBUG_PRINTLN("DataStore: primary identity/preferences migration failed; using primary storage"); - _fsExtra = nullptr; - return false; - } - enum class MigrationJournalState : uint8_t { None, Pending, Committed, Invalid, + IoError, }; static const uint8_t journal_magic[] = {'M', 'C', 'X', 'F', 1}; auto readJournal = [this]() -> MigrationJournalState { - if (!_fsExtra->exists(SECONDARY_MIGRATION_JOURNAL)) { + bool journal_exists = false; + if (!contactPathPresence(_fsExtra, SECONDARY_MIGRATION_JOURNAL, + journal_exists)) { + return MigrationJournalState::IoError; + } + if (!journal_exists) { return MigrationJournalState::None; } File journal = openRead(_fsExtra, SECONDARY_MIGRATION_JOURNAL); @@ -1416,75 +2052,156 @@ bool DataStore::migrateToSecondaryFS() { && journal.commit(); }; - auto hasSecondarySources = [this]() -> bool { + auto hasSecondarySources = [this](bool& present) -> bool { + present = false; for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) { - if (_fs->exists(to_secondary[i])) return true; + bool path_exists = false; + if (!contactPathPresence(_fs, to_secondary[i], path_exists)) return false; + if (path_exists) { + present = true; + return true; + } } for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { char path[24]; makeContactPagePath(page, path); - if (_fs->exists(path)) return true; + bool path_exists = false; + if (!contactPathPresence(_fs, path, path_exists)) return false; + if (path_exists) { + present = true; + return true; + } } for (uint8_t bucket = 0; bucket < 10; bucket++) { char path[20]; snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket); - if (_fs->exists(path)) return true; + bool path_exists = false; + if (!contactPathPresence(_fs, path, path_exists)) return false; + if (path_exists) { + present = true; + return true; + } } - return false; + return true; }; auto copySecondarySources = [this, ©]() -> bool { bool success = true; for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) { - if (!copy(_fs, _fsExtra, to_secondary[i])) success = false; + if (!copy(_fs, _fsExtra, to_secondary[i], true)) success = false; } for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { char path[24]; makeContactPagePath(page, path); - if (!copy(_fs, _fsExtra, path)) success = false; + if (!copy(_fs, _fsExtra, path, true)) success = false; } for (uint8_t bucket = 0; bucket < 10; bucket++) { char path[20]; snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket); - if (!copy(_fs, _fsExtra, path)) success = false; + if (!copy(_fs, _fsExtra, path, true)) success = false; } return success; }; auto retireSecondarySources = [this]() -> bool { + bool fixed_present[sizeof(to_secondary) / sizeof(to_secondary[0])] = {}; + uint32_t page_presence = 0; + uint16_t bucket_presence = 0; + + // Capture the complete removal set before deleting anything. A metadata + // I/O error must leave every committed primary source in place for the + // next reboot, rather than being mistaken for an absent file midway + // through cleanup. + for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) { + if (!contactPathPresence(_fs, to_secondary[i], fixed_present[i])) { + return false; + } + } + for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { + char path[24]; + makeContactPagePath(page, path); + bool path_exists = false; + if (!contactPathPresence(_fs, path, path_exists)) return false; + if (path_exists) page_presence |= 1UL << page; + } + for (uint8_t bucket = 0; bucket < 10; bucket++) { + char path[20]; + snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket); + bool path_exists = false; + if (!contactPathPresence(_fs, path, path_exists)) return false; + if (path_exists) bucket_presence |= (uint16_t)1U << bucket; + } + bool success = true; - auto retire = [this, &success](const char* path) { - if (!_fs->exists(path)) return; + auto retire = [this, &success](const char* path, bool path_exists) { + if (!path_exists) return; // The committed journal is the authority switch: every destination was // already verified before it was written. Do not compare again here, // because the secondary may legitimately advance while cleanup retries. if (!_fs->remove(path)) success = false; }; for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) { - retire(to_secondary[i]); + retire(to_secondary[i], fixed_present[i]); } for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) { char path[24]; makeContactPagePath(page, path); - retire(path); + retire(path, (page_presence & (1UL << page)) != 0); } for (uint8_t bucket = 0; bucket < 10; bucket++) { char path[20]; snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket); - retire(path); + retire(path, (bucket_presence & ((uint16_t)1U << bucket)) != 0); } return success; }; MigrationJournalState journal_state = readJournal(); - if (journal_state == MigrationJournalState::Invalid) { - MESH_DEBUG_PRINTLN("DataStore: invalid ExtraFS migration journal; using primary storage"); - _fsExtra = nullptr; + // Identity and preferences are always read from the primary filesystem. + // Recover legacy/test-layout copies before any journal result can select or + // quarantine the contact/channel store. The journal has already been read, + // however, so a failure here can never silently select the wrong authority. + if (!migratePrimarySources()) { + _contact_load_incomplete = true; + MESH_DEBUG_PRINTLN( + "DataStore: primary identity/preferences migration failed; storage quarantined until reboot"); return false; } + if (journal_state == MigrationJournalState::IoError) { + // The journal is the authority switch. If its presence cannot be + // classified, neither filesystem can safely be selected for contact or + // channel mutations this boot: Pending means primary is authoritative, + // while Committed means secondary may already be the only complete copy. + _contact_load_incomplete = true; + MESH_DEBUG_PRINTLN( + "DataStore: ExtraFS migration journal stat failed; storage quarantined until reboot"); + return false; + } + if (journal_state == MigrationJournalState::Invalid) { + // Atomic journal contents should always be one of the two valid states. + // Corruption cannot reveal whether source retirement had begun, so it has + // the same authority-unknown policy as a journal I/O error. + _contact_load_incomplete = true; + MESH_DEBUG_PRINTLN( + "DataStore: invalid ExtraFS migration journal; storage quarantined until explicit erase"); + return false; + } + + bool secondary_sources_present = false; + if (journal_state == MigrationJournalState::None + && !hasSecondarySources(secondary_sources_present)) { + // A clean post-migration boot also has no journal and no primary sources, + // so a source-stat error makes authority unknowable. Preserve both stores + // and block contact/channel use until a reboot can classify every path. + _contact_load_incomplete = true; + MESH_DEBUG_PRINTLN( + "DataStore: could not inspect primary migration sources; storage quarantined until reboot"); + return false; + } if (journal_state != MigrationJournalState::Committed - && (journal_state != MigrationJournalState::None || hasSecondarySources())) { + && (journal_state != MigrationJournalState::None + || secondary_sources_present)) { if (journal_state == MigrationJournalState::None && !writeJournal(SECONDARY_MIGRATION_PENDING)) { MESH_DEBUG_PRINTLN("DataStore: could not start ExtraFS migration transaction"); @@ -1510,8 +2227,10 @@ bool DataStore::migrateToSecondaryFS() { MESH_DEBUG_PRINTLN("DataStore: ExtraFS migration committed; source cleanup remains pending"); return false; } - if (_fsExtra->exists(SECONDARY_MIGRATION_JOURNAL) - && !_fsExtra->remove(SECONDARY_MIGRATION_JOURNAL)) { + // readJournal() found it, or writeJournal() committed it in this call, so + // a second presence probe cannot add safety. Remove the known journal + // directly after all committed sources have been retired. + if (!_fsExtra->remove(SECONDARY_MIGRATION_JOURNAL)) { MESH_DEBUG_PRINTLN("DataStore: ExtraFS migration journal cleanup failed"); return false; } @@ -1682,8 +2401,14 @@ uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_b if (findBlobInBucket(_getContactsChannelsFS(), normalized, dest_buf, length)) { return length; } + // checkAdvBlobFile() retires the old monolithic nRF cache at boot. If that + // best-effort removal failed, do not fall back to it: a durable bucket + // tombstone must remain authoritative even when its bucket is temporarily + // unreadable, and adverts are a reconstructable cache. + return 0; #endif +#if !defined(NRF52_PLATFORM) File file = openRead(_getContactsChannelsFS(), "/adv_blobs"); uint8_t len = 0; // 0 = not found if (file) { @@ -1701,6 +2426,7 @@ uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_b file.close(); } return len; +#endif } bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len) { @@ -1740,8 +2466,9 @@ bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src free(records); if (!saved) return false; if (!clearLegacyBlobRecord(_getContactsChannelsFS(), normalized)) { - MESH_DEBUG_PRINTLN("DataStore: could not retire legacy advert record"); - return false; + // The nRF reader never falls back to this reconstructable legacy cache. + MESH_DEBUG_PRINTLN( + "DataStore: advert saved; legacy cache cleanup deferred"); } return true; #else @@ -1788,16 +2515,20 @@ bool DataStore::deleteBlobByKey(const uint8_t key[], int key_len) { const uint8_t bucket = blobBucketFor(normalized); BlobRec* records = (BlobRec*)malloc(sizeof(BlobRec) * BLOB_BUCKET_SLOTS); if (records == nullptr) return false; - if (!loadBlobBucket(_getContactsChannelsFS(), bucket, records)) { + const bool bucket_valid = loadBlobBucket( + _getContactsChannelsFS(), bucket, records); + if (!bucket_valid) { memset(records, 0, sizeof(BlobRec) * BLOB_BUCKET_SLOTS); } uint8_t selected = 0; uint32_t oldest = 0xFFFFFFFFUL; + bool found = false; for (uint8_t i = 0; i < BLOB_BUCKET_SLOTS; i++) { if (memcmp(records[i].key, normalized, sizeof(records[i].key)) == 0 && (records[i].timestamp != 0 || records[i].len != 0)) { selected = i; + found = true; break; } if (records[i].timestamp < oldest) { @@ -1805,6 +2536,16 @@ bool DataStore::deleteBlobByKey(const uint8_t key[], int key_len) { selected = i; } } + const bool legacy_exists = + _getContactsChannelsFS()->exists("/adv_blobs"); + if (!legacy_exists && bucket_valid + && (!found || records[selected].len == 0)) { + // Only a successfully verified bucket can prove the key is absent or + // already tombstoned. A failed bucket read may be transient and must not + // turn a stale cached advert into an acknowledged deletion. + free(records); + return true; + } BlobRec& tombstone = records[selected]; memset(&tombstone, 0, sizeof(tombstone)); memcpy(tombstone.key, normalized, sizeof(tombstone.key)); @@ -1814,8 +2555,13 @@ bool DataStore::deleteBlobByKey(const uint8_t key[], int key_len) { free(records); if (!saved) return false; if (!clearLegacyBlobRecord(_getContactsChannelsFS(), normalized)) { - MESH_DEBUG_PRINTLN("DataStore: could not clear deleted legacy advert record"); - return false; + // The durable zero-length bucket record already masks the legacy entry, + // and checkAdvBlobFile() retires the reconstructable legacy cache on the + // next boot. The cache mutation cannot be rolled back after this commit, + // so report deletion success and keep the live contact table consistent + // with the authoritative tombstone. + MESH_DEBUG_PRINTLN( + "DataStore: deleted advert tombstoned; legacy cleanup deferred"); } return true; #else diff --git a/examples/companion_radio/DataStore.h b/examples/companion_radio/DataStore.h index adc1a2ef3..ecd3d411a 100644 --- a/examples/companion_radio/DataStore.h +++ b/examples/companion_radio/DataStore.h @@ -28,23 +28,31 @@ class DataStore { #if defined(NRF52_PLATFORM) mesh::storage::ContactSlotMap _contact_slots; mesh::storage::DirtyPageSet _dirty_contact_pages; + mesh::storage::DirtyPageSet _unread_contact_pages; + bool _contact_load_incomplete = false; + bool _identity_creation_blocked = false; + bool _primary_storage_unavailable = false; + bool _secondary_authority_unknown = false; + bool _prefs_load_incomplete = false; uint32_t _contact_page_generations[mesh::storage::CONTACT_PAGE_COUNT]; bool _legacy_contacts_pending_cleanup; bool _legacy_migration_ready; uint16_t _legacy_contact_count; bool prepareLegacyContactMigration(); - bool loadContactPages(DataStoreHost* host, uint16_t minimum_slot = 0); + bool loadContactPages(DataStoreHost* host, uint16_t minimum_slot, + uint32_t expected_page_mask); bool writeContactPage(DataStoreHost* host, uint8_t page, bool (*filter)(const ContactInfo& c)); bool truncateLegacyContacts(uint16_t remaining_contacts); - void resetContactPageState(); + void resetContactPageState(bool clear_incomplete = false); #if defined(EXTRAFS) && !defined(QSPIFLASH) bool reinitializeInternalExtraFS(); #endif #endif - void loadPrefsInt(const char *filename, CompanionNodePrefs& prefs, double& node_lat, double& node_lon); + bool loadPrefsInt(const char *filename, CompanionNodePrefs& prefs, + double& node_lat, double& node_lon); #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) void checkAdvBlobFile(); #endif @@ -57,18 +65,23 @@ public: bool repairInternalExtraFS(); FILESYSTEM* getPrimaryFS() const { return _fs; } FILESYSTEM* getSecondaryFS() const { return _fsExtra; } - void disableSecondaryFS() { _fsExtra = nullptr; } + void markPrimaryFSUnavailable(); + void disableSecondaryFS(bool authority_unknown = true); bool loadMainIdentity(mesh::LocalIdentity &identity); + bool canCreateMainIdentity() const; bool saveMainIdentity(const mesh::LocalIdentity &identity); - void loadPrefs(CompanionNodePrefs& prefs, double& node_lat, double& node_lon); + bool loadPrefs(CompanionNodePrefs& prefs, double& node_lat, + double& node_lon); bool savePrefs(const CompanionNodePrefs& prefs, double node_lat, double node_lon); void loadContacts(DataStoreHost* host); bool saveContacts(DataStoreHost* host, bool (*filter)(const ContactInfo& c) = NULL); bool markContactDirty(const ContactInfo& contact); bool releaseContact(const ContactInfo& contact); + bool restoreContactSlot(const ContactInfo& contact, uint16_t slot); bool serviceContactWrites(DataStoreHost* host, bool (*filter)(const ContactInfo& c) = NULL); bool flushContactWrites(DataStoreHost* host, bool (*filter)(const ContactInfo& c) = NULL); bool hasPendingContactWrites() const; + bool hasIncompleteContactLoad() const; void loadChannels(DataStoreHost* host); bool saveChannels(DataStoreHost* host); bool migrateToSecondaryFS(); diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index 8a12c9517..25ff1d790 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -13,6 +13,11 @@ #include "helpers/radiolib/RxBoostedGainDefaults.h" #include "helpers/radiolib/CadTiming.h" +#if defined(MESHCORE_EXTRAFS_HIL) \ + && (!defined(NRF52_PLATFORM) || !defined(EXTRAFS) || defined(QSPIFLASH)) +#error "MESHCORE_EXTRAFS_HIL requires an nRF52 internal ExtraFS build" +#endif + #if COMPANION_FEATURE_MEMORY_DIAGNOSTICS #include #endif @@ -148,6 +153,7 @@ static constexpr uint8_t DEFAULT_FEM_RX_GAIN = 1; #define RESP_CODE_CONTACTS_START 2 // first reply to CMD_GET_CONTACTS #define RESP_CODE_CONTACT 3 // multiple of these (after CMD_GET_CONTACTS) #define RESP_CODE_END_OF_CONTACTS 4 // last reply to CMD_GET_CONTACTS +#define CONTACT_STREAM_FRAME_INTERVAL_MS 5 // let slow USB hosts drain accepted frames #define RESP_CODE_SELF_INFO 5 // reply to CMD_APP_START #define RESP_CODE_SENT 6 // reply to CMD_SEND_TXT_MSG #define RESP_CODE_CONTACT_MSG_RECV 7 // a reply to CMD_SYNC_NEXT_MESSAGE (ver < 3) @@ -309,7 +315,7 @@ void MyMesh::writeDisabledFrame() { _serial->writeFrame(buf, 1); } -void MyMesh::writeContactRespFrame(uint8_t code, const ContactInfo &contact) { +bool MyMesh::writeContactRespFrame(uint8_t code, const ContactInfo &contact) { int i = 0; out_frame[i++] = code; memcpy(&out_frame[i], contact.id.pub_key, PUB_KEY_SIZE); @@ -329,18 +335,49 @@ void MyMesh::writeContactRespFrame(uint8_t code, const ContactInfo &contact) { i += 4; memcpy(&out_frame[i], &contact.lastmod, 4); i += 4; - _serial->writeFrame(out_frame, i); + return _serial->writeFrame(out_frame, i) == (size_t)i; } void MyMesh::stopContactsIterator() { if (!_iter_started) return; _iter_started = false; + _iter_start_pending = false; + _iter_contact_pending = false; + _iter_next_frame_at = 0; if (_serial != NULL) _serial->unlockReplyRoute(); } -void MyMesh::updateContactFromFrame(ContactInfo &contact, uint32_t& last_mod, const uint8_t *frame, int len) { - int i = 0; - uint8_t code = frame[i++]; // eg. CMD_ADD_UPDATE_CONTACT +static constexpr int CONTACT_UPDATE_FRAME_MIN_LEN = + 1 + PUB_KEY_SIZE + 1 + 1 + 1 + MAX_PATH_SIZE + 32 + 4; +static constexpr int CONTACT_UPDATE_FRAME_GPS_LEN = + CONTACT_UPDATE_FRAME_MIN_LEN + 8; +static constexpr int CONTACT_UPDATE_FRAME_LASTMOD_LEN = + CONTACT_UPDATE_FRAME_GPS_LEN + 4; + +static bool isPersistentContactType(uint8_t type) { + return type == ADV_TYPE_CHAT || type == ADV_TYPE_REPEATER + || type == ADV_TYPE_ROOM || type == ADV_TYPE_SENSOR; +} + +bool MyMesh::updateContactFromFrame(ContactInfo &contact, uint32_t& last_mod, const uint8_t *frame, int len) { + if (frame == NULL || len < CONTACT_UPDATE_FRAME_MIN_LEN + || (len > CONTACT_UPDATE_FRAME_MIN_LEN + && len < CONTACT_UPDATE_FRAME_GPS_LEN) + || (len > CONTACT_UPDATE_FRAME_GPS_LEN + && len < CONTACT_UPDATE_FRAME_LASTMOD_LEN) + || frame[0] != CMD_ADD_UPDATE_CONTACT) { + return false; + } + + const uint8_t type = frame[1 + PUB_KEY_SIZE]; + const uint8_t out_path_len = frame[1 + PUB_KEY_SIZE + 1 + 1]; + if (!isPersistentContactType(type) + || (out_path_len != OUT_PATH_UNKNOWN + && !mesh::Packet::isValidPathLen(out_path_len))) { + return false; + } + + int i = 1; // command byte memcpy(contact.id.pub_key, &frame[i], PUB_KEY_SIZE); i += PUB_KEY_SIZE; contact.type = frame[i++]; @@ -349,18 +386,20 @@ void MyMesh::updateContactFromFrame(ContactInfo &contact, uint32_t& last_mod, co memcpy(contact.out_path, &frame[i], MAX_PATH_SIZE); i += MAX_PATH_SIZE; memcpy(contact.name, &frame[i], 32); + contact.name[sizeof(contact.name) - 1] = 0; i += 32; memcpy(&contact.last_advert_timestamp, &frame[i], 4); i += 4; - if (len >= i + 8) { // optional fields + if (len >= CONTACT_UPDATE_FRAME_GPS_LEN) { // optional fields memcpy(&contact.gps_lat, &frame[i], 4); i += 4; memcpy(&contact.gps_lon, &frame[i], 4); i += 4; - if (len >= i + 4) { + if (len >= CONTACT_UPDATE_FRAME_LASTMOD_LEN) { memcpy(&last_mod, &frame[i], 4); } } + return true; } bool MyMesh::Frame::isChannelMsg() const { @@ -593,15 +632,36 @@ uint8_t MyMesh::getAutoAddMaxHops() const { return _prefs.autoadd_max_hops; } -void MyMesh::onContactOverwrite(const uint8_t* pub_key) { - ContactInfo* contact = lookupContactByPubKey(pub_key, PUB_KEY_SIZE); - if (contact) scheduleContactWriteAfterRelease(*contact); - _store->deleteBlobByKey(pub_key, PUB_KEY_SIZE); // delete from storage +bool MyMesh::canMutateContacts() const { + return !_store->hasIncompleteContactLoad(); +} + +bool MyMesh::onContactOverwrite(const ContactInfo& contact) { + uint16_t released_slot = mesh::storage::CONTACT_SLOT_NONE; + if (!scheduleContactWriteAfterRelease(contact, released_slot)) { + MESH_DEBUG_PRINTLN( + "Contact overwrite refused: persistent slot release failed"); + return false; + } + if (!_store->deleteBlobByKey(contact.id.pub_key, PUB_KEY_SIZE)) { + // releaseContact() is failure-atomic and this single-threaded path still + // owns the just-freed slot. Put it back before refusing the overwrite so a + // full filesystem cannot silently retain stale advert state for a contact + // which the live table has already replaced. + if (!restoreContactWriteAfterRelease(contact, released_slot)) { + MESH_DEBUG_PRINTLN( + "Contact overwrite rollback failed after advert-cache error"); + } + MESH_DEBUG_PRINTLN( + "Contact overwrite refused: advert-cache deletion failed"); + return false; + } if (_serial != NULL && _serial->isConnected()) { out_frame[0] = PUSH_CODE_CONTACT_DELETED; - memcpy(&out_frame[1], pub_key, PUB_KEY_SIZE); + memcpy(&out_frame[1], contact.id.pub_key, PUB_KEY_SIZE); _serial->writeFrame(out_frame, 1 + PUB_KEY_SIZE); } + return true; } void MyMesh::onContactsFull() { @@ -1451,6 +1511,11 @@ MyMesh::MyMesh(mesh::Radio &radio, mesh::RNG &rng, mesh::RTCClock &rtc, SimpleMe _serial(NULL), _mota_source_control(NULL), telemetry(MAX_PACKET_PAYLOAD - 4), _store(&store), _ui(ui), _iter(0) { _iter_started = false; + _iter_start_pending = false; + _iter_contact_pending = false; + _iter_next_frame_at = 0; + _iter_total_count = 0; + _iter_table_revision = 0; _cli_rescue = false; #ifdef ENABLE_USB_INTERFACE _terminal_mode = false; @@ -1586,19 +1651,27 @@ void MyMesh::begin(bool has_display, bool radio_available) { initializeOfflineQueue(); BaseChatMesh::begin(); - const bool is_new_install = !_store->loadMainIdentity(self_id) + const bool identity_loaded = _store->loadMainIdentity(self_id); + const bool is_new_install = !identity_loaded || mesh::hasReservedIdentityPrefix(self_id); + if (is_new_install && !_store->canCreateMainIdentity()) { + MESH_DEBUG_PRINTLN( + "Identity unavailable from persistent storage; rebooting without replacement"); + board.reboot(); + return; + } + bool identity_ready = true; if (is_new_install) { identity_ready = mesh::generateUsableLocalIdentity(self_id, radio_new_identity); - if (identity_ready) _store->saveMainIdentity(self_id); + if (identity_ready) identity_ready = _store->saveMainIdentity(self_id); } #if defined(ESP32_PLATFORM) mesh::discardESP32TrueRandom(); #endif if (!identity_ready) { - MESH_DEBUG_PRINTLN("Identity generation exhausted all attempts; rebooting"); + MESH_DEBUG_PRINTLN("Identity generation/save failed; rebooting"); board.reboot(); return; } @@ -1625,7 +1698,8 @@ void MyMesh::begin(bool has_display, bool radio_available) { #endif // load persisted prefs - _store->loadPrefs(_prefs, sensors.node_lat, sensors.node_lon); + const bool prefs_ready = + _store->loadPrefs(_prefs, sensors.node_lat, sensors.node_lon); // v1.17.1.2 repairs the Companion default-off regression for both fresh // installs and devices that already persisted the regressed value. The @@ -1693,8 +1767,9 @@ void MyMesh::begin(bool has_display, bool radio_available) { recalcRxPowerSavingFromLevel(_prefs.rx_ps_level, _prefs.sf, _prefs.bw, _prefs.rx_ps_preamble, &_prefs.rx_ps_rx_us, &_prefs.rx_ps_sleep_us); - if (power_saving_default_migrated || bluetooth_name_repaired - || display_rotation_repaired) { + if (prefs_ready + && (power_saving_default_migrated || bluetooth_name_repaired + || display_rotation_repaired)) { _store->savePrefs(_prefs, sensors.node_lat, sensors.node_lon); } #if MESH_USB_LOGGING_AVAILABLE @@ -3502,7 +3577,15 @@ void MyMesh::handleCmdFrame(size_t len) { text[tlen] = 0; // ensure null reply_buf[0] = 0; - if (!handleCommand(text, 0, reply_buf)) { + bool handled = false; +#if defined(MESHCORE_EXTRAFS_HIL) + // This hook is deliberately reachable only from a directly attached + // Companion frame. It is not part of handleCommand(), so an on-air + // administrator command can never fill, corrupt, or reset the device. + handled = handleExtraFsHilCommand(text, reply_buf, sizeof(reply_buf)); +#endif + if (!handled) handled = handleCommand(text, 0, reply_buf); + if (!handled) { strcat(reply_buf, "Unknown command"); // reply_buf may have cmd prefix from 'text' } out_frame[0] = RESP_CODE_CLI_REPLY; @@ -3680,6 +3763,10 @@ void MyMesh::handleCmdFrame(size_t len) { } else if (cmd_frame[0] == CMD_GET_CONTACTS) { // get Contact list if (_iter_started) { writeErrFrame(ERR_CODE_BAD_STATE); // iterator is currently busy + } else if (_store->hasIncompleteContactLoad()) { + // Never present a transiently unread page as an authoritative partial + // contact list. A reboot can retry the untouched source page. + writeErrFrame(ERR_CODE_FILE_IO_ERROR); } else { if (len >= 5) { // has optional 'since' param memcpy(&_iter_filter_since, &cmd_frame[1], 4); @@ -3692,13 +3779,20 @@ void MyMesh::handleCmdFrame(size_t len) { _serial->lockReplyRoute(); _iter = startContactsIterator(); _iter_started = true; + _iter_start_pending = true; + _iter_contact_pending = false; + _iter_next_frame_at = 0; + _iter_total_count = getNumContacts(); + _iter_table_revision = getContactTableRevision(); _most_recent_lastmod = 0; uint8_t reply[5]; reply[0] = RESP_CODE_CONTACTS_START; - uint32_t count = getNumContacts(); // total, NOT filtered count - memcpy(&reply[1], &count, 4); - _serial->writeFrame(reply, 5); + memcpy(&reply[1], &_iter_total_count, 4); // total, NOT filtered count + if (_serial->writeFrame(reply, 5) == 5) { + _iter_start_pending = false; + _iter_next_frame_at = futureMillis(CONTACT_STREAM_FRAME_INTERVAL_MS); + } } } else if (cmd_frame[0] == CMD_SET_ADVERT_NAME && len >= 2) { int nlen = len - 1; @@ -3762,52 +3856,123 @@ void MyMesh::handleCmdFrame(size_t len) { writeErrFrame(ERR_CODE_TABLE_FULL); } } else if (cmd_frame[0] == CMD_RESET_PATH && len >= 1 + 32) { - uint8_t *pub_key = &cmd_frame[1]; - ContactInfo *recipient = lookupContactByPubKey(pub_key, PUB_KEY_SIZE); - if (recipient) { - recipient->out_path_len = OUT_PATH_UNKNOWN; - // recipient->lastmod = ?? shouldn't be needed, app already has this version of contact - scheduleContactWrite(*recipient); - writeOKFrame(); + if (_store->hasIncompleteContactLoad()) { + writeErrFrame(ERR_CODE_FILE_IO_ERROR); } else { - writeErrFrame(ERR_CODE_NOT_FOUND); // unknown contact - } - } else if (cmd_frame[0] == CMD_ADD_UPDATE_CONTACT && len >= 1 + 32 + 2 + 1) { - uint8_t *pub_key = &cmd_frame[1]; - ContactInfo *recipient = lookupContactByPubKey(pub_key, PUB_KEY_SIZE); - uint32_t last_mod = getRTCClock()->getCurrentTime(); // fallback value if not present in cmd_frame - if (recipient) { - updateContactFromFrame(*recipient, last_mod, cmd_frame, len); - recipient->lastmod = last_mod; - scheduleContactWrite(*recipient); - updateGpsTelemetryPolicy(); - writeOKFrame(); - } else { - ContactInfo contact; - updateContactFromFrame(contact, last_mod, cmd_frame, len); - contact.lastmod = last_mod; - contact.sync_since = 0; - if (addContact(contact)) { - ContactInfo* added = lookupContactByPubKey(contact.id.pub_key, PUB_KEY_SIZE); - if (added) scheduleContactWrite(*added); - updateGpsTelemetryPolicy(); - writeOKFrame(); + uint8_t *pub_key = &cmd_frame[1]; + ContactInfo *recipient = lookupPersistentContactByPubKey(pub_key, PUB_KEY_SIZE); + if (recipient) { + const uint8_t previous_out_path_len = recipient->out_path_len; + recipient->out_path_len = OUT_PATH_UNKNOWN; + // recipient->lastmod = ?? shouldn't be needed, app already has this version of contact + if (scheduleContactWrite(*recipient)) { + ContactInfo* transient = lookupTransientContactByPubKey( + pub_key, PUB_KEY_SIZE); + if (transient != NULL) clearTransientContact(*transient); + writeOKFrame(); + } else { + recipient->out_path_len = previous_out_path_len; + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } } else { - writeErrFrame(ERR_CODE_TABLE_FULL); + writeErrFrame(ERR_CODE_NOT_FOUND); // unknown contact + } + } + } else if (cmd_frame[0] == CMD_ADD_UPDATE_CONTACT) { + if (len < CONTACT_UPDATE_FRAME_MIN_LEN) { + writeErrFrame(ERR_CODE_ILLEGAL_ARG); + } else if (_store->hasIncompleteContactLoad()) { + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } else { + uint8_t *pub_key = &cmd_frame[1]; + ContactInfo *recipient = lookupPersistentContactByPubKey(pub_key, PUB_KEY_SIZE); + ContactInfo candidate = {}; + if (recipient != NULL) candidate = *recipient; + + uint32_t last_mod = getRTCClock()->getCurrentTime(); // fallback value if not present in cmd_frame + if (!updateContactFromFrame(candidate, last_mod, cmd_frame, len)) { + writeErrFrame(ERR_CODE_ILLEGAL_ARG); + } else { + candidate.lastmod = last_mod; + if (recipient == NULL) candidate.sync_since = 0; + + ContactInfo* transient = lookupTransientContactByPubKey( + pub_key, PUB_KEY_SIZE); + ContactInfo previous_transient = {}; + if (transient != NULL) previous_transient = *transient; + + if (recipient != NULL) { + const ContactInfo previous = *recipient; + *recipient = candidate; + if (scheduleContactWrite(*recipient)) { + // A prior anonymous request can leave a same-key entry in the + // reserved prefix. Retire it in place; never compact that prefix. + if (transient != NULL) clearTransientContact(*transient); + updateGpsTelemetryPolicy(); + writeOKFrame(); + } else { + *recipient = previous; + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } + } else { + // Promotion must free the exact transient prefix slot before lookup + // of the newly-added persistent record. Restore it on every failure. + if (transient != NULL) clearTransientContact(*transient); + if (addContact(candidate)) { + ContactInfo* added = lookupPersistentContactByPubKey( + candidate.id.pub_key, PUB_KEY_SIZE); + if (added != NULL && scheduleContactWrite(*added)) { + updateGpsTelemetryPolicy(); + writeOKFrame(); + } else { + if (added != NULL) removeContact(*added); + if (transient != NULL) *transient = previous_transient; + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } + } else { + if (transient != NULL) *transient = previous_transient; + writeErrFrame(ERR_CODE_TABLE_FULL); + } + } } } } else if (cmd_frame[0] == CMD_REMOVE_CONTACT && len >= 1 + PUB_KEY_SIZE) { - uint8_t *pub_key = &cmd_frame[1]; - ContactInfo *recipient = lookupContactByPubKey(pub_key, PUB_KEY_SIZE); - ContactInfo removed; - if (recipient) removed = *recipient; - if (recipient && removeContact(*recipient)) { - scheduleContactWriteAfterRelease(removed); - _store->deleteBlobByKey(pub_key, PUB_KEY_SIZE); - updateGpsTelemetryPolicy(); - writeOKFrame(); + if (_store->hasIncompleteContactLoad()) { + writeErrFrame(ERR_CODE_FILE_IO_ERROR); } else { - writeErrFrame(ERR_CODE_NOT_FOUND); // not found, or unable to remove + uint8_t *pub_key = &cmd_frame[1]; + ContactInfo *recipient = lookupPersistentContactByPubKey(pub_key, PUB_KEY_SIZE); + if (recipient == NULL) { + writeErrFrame(ERR_CODE_NOT_FOUND); + } else { + ContactInfo removed = *recipient; + // releaseContact() is failure-atomic, so reserve the persistent deletion + // before changing the live table. A failed release must not be ACKed. + uint16_t released_slot = mesh::storage::CONTACT_SLOT_NONE; + if (!scheduleContactWriteAfterRelease(removed, released_slot)) { + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } else if (!_store->deleteBlobByKey(pub_key, PUB_KEY_SIZE)) { + // Keep the live contact and restore ownership of its persistent slot. + // A full filesystem must not turn a failed cache deletion into an + // acknowledged contact removal with stale share data left behind. + if (!restoreContactWriteAfterRelease(*recipient, released_slot)) { + MESH_DEBUG_PRINTLN( + "Contact removal rollback failed after advert-cache error"); + } + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } else if (!removeContact(*recipient)) { + // The lookup above makes this unreachable in the single-threaded mesh + // loop, but restore persistence bookkeeping if that invariant changes. + restoreContactWriteAfterRelease(*recipient, released_slot); + writeErrFrame(ERR_CODE_BAD_STATE); + } else { + ContactInfo* transient = lookupTransientContactByPubKey( + pub_key, PUB_KEY_SIZE); + if (transient != NULL) clearTransientContact(*transient); + updateGpsTelemetryPolicy(); + writeOKFrame(); + } + } } } else if (cmd_frame[0] == CMD_SHARE_CONTACT && len >= 1 + PUB_KEY_SIZE) { uint8_t *pub_key = &cmd_frame[1]; @@ -3860,7 +4025,12 @@ void MyMesh::handleCmdFrame(size_t len) { } } } else if (cmd_frame[0] == CMD_IMPORT_CONTACT && len > 2 + 32 + 64) { - if (importContact(&cmd_frame[1], len - 1)) { + if (_store->hasIncompleteContactLoad()) { + // importContact() is asynchronous: without this admission gate the + // queued advert would be ACKed here and then deliberately discarded by + // BaseChatMesh's incomplete-table mutation veto on the next loop. + writeErrFrame(ERR_CODE_FILE_IO_ERROR); + } else if (importContact(&cmd_frame[1], len - 1)) { updateGpsTelemetryPolicy(); writeOKFrame(); } else { @@ -4308,15 +4478,23 @@ void MyMesh::handleCmdFrame(size_t len) { writeErrFrame(ERR_CODE_UNSUPPORTED_CMD); // not supported (yet) } else if (cmd_frame[0] == CMD_SET_CHANNEL && len >= 2 + 32 + 16) { uint8_t channel_idx = cmd_frame[1]; + ChannelDetails previous; ChannelDetails channel; StrHelper::strncpy(channel.name, (char *)&cmd_frame[2], 32); memset(channel.channel.secret, 0, sizeof(channel.channel.secret)); memcpy(channel.channel.secret, &cmd_frame[2 + 32], 16); // NOTE: only 128-bit supported - if (setChannel(channel_idx, channel)) { - saveChannels(); + if (!getChannel(channel_idx, previous) + || !setChannel(channel_idx, channel)) { + writeErrFrame(ERR_CODE_NOT_FOUND); // bad channel_idx + } else if (saveChannels()) { writeOKFrame(); } else { - writeErrFrame(ERR_CODE_NOT_FOUND); // bad channel_idx + // AtomicFileWriter preserves the last complete channels file. Keep the + // live table consistent with it when a full or failing filesystem cannot + // commit this update, and never acknowledge a change that will disappear + // after reboot. + setChannel(channel_idx, previous); + writeErrFrame(ERR_CODE_FILE_IO_ERROR); } } else if (cmd_frame[0] == CMD_SIGN_START) { BaseSerialInterface* signing_route = _serial->captureReplyRoute(); @@ -4890,20 +5068,48 @@ bool MyMesh::prepareForUiShutdown() { return flushContactsBeforeReboot(); } -void MyMesh::scheduleContactWrite(const ContactInfo& contact) { - if (_store->markContactDirty(contact)) { - mesh::scheduleLazyPersistenceMutation( - dirty_contacts_expiry, dirty_contacts_failures, - futureMillis(LAZY_CONTACTS_WRITE_DELAY)); +bool MyMesh::scheduleContactWrite(const ContactInfo& contact) { + // Anonymous contacts live only in the reserved transient prefix and are + // deliberately excluded from contact storage. Never allocate a persistent + // slot for one when a path/message callback updates its runtime state. + if (contact.type == ADV_TYPE_NONE || isTransientContact(contact)) return true; + if (!_store->markContactDirty(contact)) { + MESH_DEBUG_PRINTLN( + "Contact persistence scheduling failed: no persistent slot"); + return false; } + mesh::scheduleLazyPersistenceMutation( + dirty_contacts_expiry, dirty_contacts_failures, + futureMillis(LAZY_CONTACTS_WRITE_DELAY)); + return true; } -void MyMesh::scheduleContactWriteAfterRelease(const ContactInfo& contact) { - if (_store->releaseContact(contact)) { - mesh::scheduleLazyPersistenceMutation( - dirty_contacts_expiry, dirty_contacts_failures, - futureMillis(LAZY_CONTACTS_WRITE_DELAY)); +bool MyMesh::scheduleContactWriteAfterRelease(const ContactInfo& contact, + uint16_t& released_slot) { +#if defined(NRF52_PLATFORM) + released_slot = contact.storage_slot; +#else + released_slot = mesh::storage::CONTACT_SLOT_NONE; +#endif + if (!_store->releaseContact(contact)) { + MESH_DEBUG_PRINTLN( + "Contact removal scheduling failed: persistent slot was not reserved"); + return false; } + mesh::scheduleLazyPersistenceMutation( + dirty_contacts_expiry, dirty_contacts_failures, + futureMillis(LAZY_CONTACTS_WRITE_DELAY)); + return true; +} + +bool MyMesh::restoreContactWriteAfterRelease(const ContactInfo& contact, + uint16_t released_slot) { + if (!_store->restoreContactSlot(contact, released_slot)) { + MESH_DEBUG_PRINTLN( + "Contact removal rollback failed: released slot was not restored"); + return false; + } + return true; } #if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH) @@ -4912,11 +5118,25 @@ void MyMesh::repairInternalExtraFS(Stream& output) { // active this is a non-destructive retry of any incomplete migration, not a // format. Do not overwrite it from RAM until every authoritative source file // has been copied, verified, and retired from primary storage. + const bool authoritative_reload_required = + _store->hasIncompleteContactLoad(); if (!_store->repairInternalExtraFS()) { output.println(" Error: internal ExtraFS repair/migration failed; retry repair extrafs"); return; } + if (authoritative_reload_required) { + // This boot intentionally loaded no/partial contacts while secondary + // authority was unknown. The repair has made the filesystem available, + // but rebuilding it from that RAM image would destroy the data just + // recovered. Reboot and reload the verified store before permitting any + // normal persistence. + output.println( + " > internal ExtraFS repaired; rebooting to reload authoritative data"); + _scheduled_reboot_at = futureMillis(1000); + return; + } + const bool contacts_saved = _store->saveContacts(this, save_filter); const bool channels_saved = saveChannels(); const bool capacity_ready = _store->getStorageTotalKb() == 100; @@ -4936,6 +5156,440 @@ void MyMesh::repairInternalExtraFS(Stream& output) { output.println(" Error: ExtraFS is active, but data rebuild/verification failed; retry repair extrafs"); } } + +#if defined(MESHCORE_EXTRAFS_HIL) +bool MyMesh::handleExtraFsHilCommand(const char* command, char* reply, + size_t reply_size) { + static const char* const fill_path = "/__hil.fill"; + static const char* const contact_failure_marker = "/__hil.readfail"; + static const uint8_t contact_stat_failure_flag = 0x80; + static const char* const prefix = "hil extrafs "; + if (strncmp(command, prefix, strlen(prefix)) != 0) return false; + + FILESYSTEM* fs = _store->getSecondaryFS(); + const char* action = command + strlen(prefix); + auto armContactPageFailure = [fs](uint8_t encoded_failure) -> int8_t { + if (fs == nullptr) return 0; + // Do not replace an unread/read-failure request with a stat-failure request + // (or vice versa). One marker allocation represents exactly one fault. + if (fs->exists(contact_failure_marker)) return -1; + + File marker = fs->open(contact_failure_marker, FILE_O_WRITE); + const bool written = marker + && marker.write(&encoded_failure, 1) == 1; + if (marker) { + marker.flush(); + marker.close(); + } + File verify = fs->open(contact_failure_marker, FILE_O_READ); + const bool durable = written && verify && verify.size() == 1 + && verify.read() == encoded_failure; + if (verify) verify.close(); + if (!durable && fs->exists(contact_failure_marker)) { + fs->remove(contact_failure_marker); + } + return durable ? 1 : 0; + }; + auto findHilContact = [this](const char* encoded, + size_t encoded_len) -> ContactInfo* { + // Seven bytes match the advert-cache key width and keep the direct-only + // HIL command below the Companion CLI frame limit. + if (encoded_len != 14) return nullptr; + char text[15]; + memcpy(text, encoded, encoded_len); + text[encoded_len] = 0; + uint8_t key_prefix[7]; + if (!mesh::Utils::fromHex(key_prefix, sizeof(key_prefix), text)) { + return nullptr; + } + return lookupPersistentContactByPubKey( + key_prefix, sizeof(key_prefix)); + }; + if (strcmp(action, "status") == 0) { + uint32_t fill_size = 0; + if (fs != nullptr) { + File fill = fs->open(fill_path, FILE_O_READ); + if (fill) { + fill_size = fill.size(); + fill.close(); + } + } + snprintf(reply, reply_size, + "HIL ExtraFS active=%u used=%luKiB total=%luKiB fill=%lu", + fs != nullptr ? 1U : 0U, + (unsigned long)_store->getStorageUsedKb(), + (unsigned long)_store->getStorageTotalKb(), + (unsigned long)fill_size); + return true; + } + + if (strcmp(action, "clear") == 0) { + if (fs == nullptr) { + snprintf(reply, reply_size, "HIL ExtraFS unavailable"); + } else if ((!fs->exists(fill_path) || fs->remove(fill_path)) + && !fs->exists(fill_path)) { + snprintf(reply, reply_size, "HIL ExtraFS filler cleared"); + } else { + snprintf(reply, reply_size, "HIL ExtraFS filler clear failed"); + } + return true; + } + + static const char* const contact_slot_prefix = "contact-slot "; + if (strncmp(action, contact_slot_prefix, + strlen(contact_slot_prefix)) == 0) { + const char* encoded = action + strlen(contact_slot_prefix); + ContactInfo* contact = findHilContact(encoded, strlen(encoded)); + if (contact == nullptr) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs contact-slot <14 hex chars>"); + } else { + snprintf(reply, reply_size, "HIL contact slot=%u", + (unsigned)contact->storage_slot); + } + return true; + } + + static const char* const seed_advert_prefix = "seed-advert "; + if (strncmp(action, seed_advert_prefix, + strlen(seed_advert_prefix)) == 0) { + const char* encoded = action + strlen(seed_advert_prefix); + static const char* const confirm = " CONFIRM"; + const size_t value_len = strlen(encoded); + const size_t confirm_len = strlen(confirm); + ContactInfo* contact = value_len == 14 + confirm_len + && strcmp(encoded + 14, confirm) == 0 + ? findHilContact(encoded, 14) : nullptr; + if (contact == nullptr || fs == nullptr) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs seed-advert <14 hex chars> CONFIRM"); + return true; + } + + uint8_t advert[PUB_KEY_SIZE + 4 + SIGNATURE_SIZE]; + memset(advert, 0xA5, sizeof(advert)); + memcpy(advert, contact->id.pub_key, PUB_KEY_SIZE); + uint8_t verify[MAX_PACKET_PAYLOAD]; + const bool saved = _store->putBlobByKey( + contact->id.pub_key, PUB_KEY_SIZE, advert, sizeof(advert)); + const uint8_t stored = saved ? _store->getBlobByKey( + contact->id.pub_key, PUB_KEY_SIZE, verify) : 0; + if (!saved || stored != sizeof(advert) + || memcmp(advert, verify, sizeof(advert)) != 0) { + snprintf(reply, reply_size, "HIL advert seed failed"); + } else { + snprintf(reply, reply_size, "HIL advert seeded slot=%u", + (unsigned)contact->storage_slot); + } + return true; + } + + static const char* const clear_advert_prefix = "clear-advert "; + if (strncmp(action, clear_advert_prefix, + strlen(clear_advert_prefix)) == 0) { + const char* encoded = action + strlen(clear_advert_prefix); + static const char* const confirm = " CONFIRM"; + const size_t value_len = strlen(encoded); + const size_t confirm_len = strlen(confirm); + ContactInfo* contact = value_len == 14 + confirm_len + && strcmp(encoded + 14, confirm) == 0 + ? findHilContact(encoded, 14) : nullptr; + if (contact == nullptr || fs == nullptr) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs clear-advert <14 hex chars> CONFIRM"); + } else if (!_store->deleteBlobByKey( + contact->id.pub_key, PUB_KEY_SIZE)) { + snprintf(reply, reply_size, "HIL advert clear failed"); + } else { + uint8_t verify[MAX_PACKET_PAYLOAD]; + const uint8_t stored = _store->getBlobByKey( + contact->id.pub_key, PUB_KEY_SIZE, verify); + snprintf(reply, reply_size, stored == 0 + ? "HIL advert cleared" : "HIL advert clear verify failed"); + } + return true; + } + + static const char* const fail_stat_prefix = "fail-stat-page "; + if (strncmp(action, fail_stat_prefix, strlen(fail_stat_prefix)) == 0) { + const char* value = action + strlen(fail_stat_prefix); + char* end = nullptr; + const unsigned long page = strtoul(value, &end, 10); + char page_path[24]; + snprintf(page_path, sizeof(page_path), "/contacts4_%02lu", page); + if (value == end || strcmp(end, " CONFIRM") != 0 + || page >= mesh::storage::CONTACT_PAGE_COUNT || fs == nullptr + || !fs->exists(page_path) || _store->hasPendingContactWrites() + || _store->hasIncompleteContactLoad()) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs fail-stat-page <0..%u> CONFIRM", + (unsigned)(mesh::storage::CONTACT_PAGE_COUNT - 1)); + return true; + } + const int8_t armed = armContactPageFailure( + contact_stat_failure_flag | static_cast(page)); + if (armed < 0) { + snprintf(reply, reply_size, "HIL contact failure already armed"); + } else if (armed == 0) { + snprintf(reply, reply_size, "HIL stat-failure marker write failed"); + } else { + snprintf(reply, reply_size, + "HIL contact page %lu stat failure armed", page); + } + return true; + } + + static const char* const fail_read_prefix = "fail-read-page "; + if (strncmp(action, fail_read_prefix, strlen(fail_read_prefix)) == 0) { + const char* value = action + strlen(fail_read_prefix); + char* end = nullptr; + const unsigned long page = strtoul(value, &end, 10); + char page_path[24]; + snprintf(page_path, sizeof(page_path), "/contacts4_%02lu", page); + if (value == end || strcmp(end, " CONFIRM") != 0 + || page >= mesh::storage::CONTACT_PAGE_COUNT || fs == nullptr + || !fs->exists(page_path) || _store->hasPendingContactWrites() + || _store->hasIncompleteContactLoad()) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs fail-read-page <0..%u> CONFIRM", + (unsigned)(mesh::storage::CONTACT_PAGE_COUNT - 1)); + return true; + } + const int8_t armed = armContactPageFailure(static_cast(page)); + if (armed < 0) { + snprintf(reply, reply_size, "HIL contact failure already armed"); + } else if (armed == 0) { + snprintf(reply, reply_size, "HIL read-failure marker write failed"); + } else { + snprintf(reply, reply_size, + "HIL contact page %lu read failure armed", page); + } + return true; + } + + static const char* const fill_prefix = "fill "; + if (strncmp(action, fill_prefix, strlen(fill_prefix)) == 0) { + const char* value = action + strlen(fill_prefix); + char* end = nullptr; + const unsigned long requested = strtoul(value, &end, 10); + if (value == end || *end != 0 || requested > 128UL * 1024UL) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs fill <0..131072>"); + return true; + } + if (fs == nullptr) { + snprintf(reply, reply_size, "HIL ExtraFS unavailable"); + return true; + } + if (fs->exists(fill_path) && !fs->remove(fill_path)) { + snprintf(reply, reply_size, "HIL ExtraFS old filler remove failed"); + return true; + } + + uint8_t chunk[128]; + memset(chunk, 0xA5, sizeof(chunk)); + uint32_t committed = 0; + while (committed < requested) { + // LittleFS can accept buffered writes which are all rolled back when the + // final sync runs out of blocks. Commit and read-verify each small extent + // so the filler represents durable pressure, not an optimistic write() + // count for a file which disappears on close. + File fill = fs->open(fill_path, FILE_O_WRITE); + if (!fill || fill.size() != committed || !fill.seek(committed)) { + if (fill) fill.close(); + break; + } + + const uint32_t extent_end = committed + 1024UL < requested + ? committed + 1024UL : requested; + bool extent_written = true; + while (committed < extent_end) { + const size_t remaining = extent_end - committed; + const size_t count = remaining < sizeof(chunk) ? remaining : sizeof(chunk); + if (fill.write(chunk, count) != count) { + extent_written = false; + break; + } + committed += count; + } + fill.flush(); + fill.close(); + + File verify = fs->open(fill_path, FILE_O_READ); + const uint32_t durable_size = verify ? verify.size() : 0; + if (verify) verify.close(); + if (!extent_written || durable_size != committed) { + committed = durable_size; + break; + } + } + snprintf(reply, reply_size, + "HIL ExtraFS fill requested=%lu written=%lu used=%luKiB total=%luKiB", + requested, (unsigned long)committed, + (unsigned long)_store->getStorageUsedKb(), + (unsigned long)_store->getStorageTotalKb()); + return true; + } + + static const char* const corrupt_prefix = "corrupt-page "; + static const char* const page_mask_prefix = "page-mask "; + if (strncmp(action, page_mask_prefix, strlen(page_mask_prefix)) == 0) { + const char* value = action + strlen(page_mask_prefix); + char* end = nullptr; + const unsigned long page = strtoul(value, &end, 10); + if (value == end || *end != 0 + || page >= mesh::storage::CONTACT_PAGE_COUNT || fs == nullptr) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs page-mask <0..%u>", + (unsigned)(mesh::storage::CONTACT_PAGE_COUNT - 1)); + return true; + } + + char path[24]; + snprintf(path, sizeof(path), "/contacts4_%02lu", page); + File page_file = fs->open(path, FILE_O_READ); + uint8_t raw_header[mesh::storage::CONTACT_PAGE_HEADER_SIZE]; + mesh::storage::ContactPageHeader header = {}; + const bool valid = page_file + && page_file.read(raw_header, sizeof(raw_header)) == sizeof(raw_header) + && mesh::storage::decodeContactPageHeader( + raw_header, static_cast(page), header); + if (page_file) page_file.close(); + if (!valid) { + snprintf(reply, reply_size, "HIL contact page unavailable"); + } else { + snprintf(reply, reply_size, + "HIL contact page %lu occupied=%08lx", page, + (unsigned long)header.occupied); + } + return true; + } + + static const char* const corrupt_occupied_prefix = "corrupt-occupied "; + if (strncmp(action, corrupt_occupied_prefix, + strlen(corrupt_occupied_prefix)) == 0) { + const char* value = action + strlen(corrupt_occupied_prefix); + char* page_end = nullptr; + const unsigned long page = strtoul(value, &page_end, 10); + const char* slot_value = page_end; + while (*slot_value == ' ') slot_value++; + char* slot_end = nullptr; + const unsigned long slot = strtoul(slot_value, &slot_end, 10); + if (value == page_end || slot_value == slot_end + || strcmp(slot_end, " CONFIRM") != 0 + || page >= mesh::storage::CONTACT_PAGE_COUNT || slot >= 32 + || fs == nullptr || _store->hasPendingContactWrites() + || _store->hasIncompleteContactLoad()) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs corrupt-occupied CONFIRM"); + return true; + } + + char path[24]; + snprintf(path, sizeof(path), "/contacts4_%02lu", page); + const uint32_t byte_offset = 8 + slot / 8; + File original = fs->open(path, FILE_O_READ); + if (!original || !original.seek(byte_offset)) { + if (original) original.close(); + snprintf(reply, reply_size, "HIL contact page unavailable"); + return true; + } + const int prior = original.read(); + original.close(); + if (prior < 0) { + snprintf(reply, reply_size, "HIL contact page read failed"); + return true; + } + + const uint8_t changed = static_cast(prior) + ^ static_cast(1U << (slot % 8)); + File update = fs->open(path, FILE_O_WRITE); + if (!update || !update.seek(byte_offset) + || update.write(&changed, 1) != 1) { + if (update) update.close(); + snprintf(reply, reply_size, "HIL contact occupancy corrupt failed"); + return true; + } + update.flush(); + update.close(); + + File verify = fs->open(path, FILE_O_READ); + const bool changed_on_flash = verify && verify.seek(byte_offset) + && verify.read() == changed; + if (verify) verify.close(); + snprintf(reply, reply_size, changed_on_flash + ? "HIL contact page %lu occupancy bit %lu corrupted" + : "HIL contact occupancy corrupt verify failed", page, slot); + return true; + } + + if (strncmp(action, corrupt_prefix, strlen(corrupt_prefix)) == 0) { + const char* value = action + strlen(corrupt_prefix); + char* end = nullptr; + const unsigned long page = strtoul(value, &end, 10); + if (value == end || strcmp(end, " CONFIRM") != 0 + || page >= mesh::storage::CONTACT_PAGE_COUNT) { + snprintf(reply, reply_size, + "HIL usage: hil extrafs corrupt-page <0..%u> CONFIRM", + (unsigned)(mesh::storage::CONTACT_PAGE_COUNT - 1)); + return true; + } + if (fs == nullptr || _store->hasPendingContactWrites() + || _store->hasIncompleteContactLoad()) { + snprintf(reply, reply_size, "HIL ExtraFS unavailable or contacts pending"); + return true; + } + + char path[24]; + snprintf(path, sizeof(path), "/contacts4_%02lu", page); + File original = fs->open(path, FILE_O_READ); + if (!original || original.size() != mesh::storage::CONTACT_PAGE_FILE_SIZE + || !original.seek(mesh::storage::CONTACT_PAGE_HEADER_SIZE)) { + if (original) original.close(); + snprintf(reply, reply_size, "HIL contact page unavailable"); + return true; + } + const int prior = original.read(); + original.close(); + if (prior < 0) { + snprintf(reply, reply_size, "HIL contact page read failed"); + return true; + } + + const uint8_t changed = static_cast(prior) ^ 0xA5U; + File update = fs->open(path, FILE_O_WRITE); + if (!update || !update.seek(mesh::storage::CONTACT_PAGE_HEADER_SIZE) + || update.write(&changed, 1) != 1) { + if (update) update.close(); + snprintf(reply, reply_size, "HIL contact page corrupt failed"); + return true; + } + update.flush(); + update.close(); + + File verify = fs->open(path, FILE_O_READ); + const bool changed_on_flash = verify + && verify.seek(mesh::storage::CONTACT_PAGE_HEADER_SIZE) + && verify.read() == changed; + if (verify) verify.close(); + snprintf(reply, reply_size, changed_on_flash + ? "HIL contact page %lu CRC corrupted" + : "HIL contact page corrupt verify failed", page); + return true; + } + + if (strcmp(action, "unsafe-reset CONFIRM") == 0) { + // Intentionally bypass all persistence hooks so HIL can prove that a + // mutation was already committed before an event-driven sleep. + NVIC_SystemReset(); + while (true) {} + } + + snprintf(reply, reply_size, + "HIL usage: hil extrafs "); + return true; +} +#endif #endif bool MyMesh::applyAndSaveFemRxGain(bool enabled) { @@ -5465,6 +6119,7 @@ void MyMesh::handleTerminalPath(ContactInfo& recipient, return; } + const ContactInfo previous = recipient; memset(recipient.out_path, 0, sizeof(recipient.out_path)); if (path.mode == mesh::cli::TerminalPathMode::Clear) { recipient.out_path_len = OUT_PATH_UNKNOWN; @@ -5472,7 +6127,12 @@ void MyMesh::handleTerminalPath(ContactInfo& recipient, recipient.out_path_len = mesh::Packet::copyPath( recipient.out_path, _terminal_tmp_buf, path.encoded_len); } - scheduleContactWrite(recipient); + if (!scheduleContactWrite(recipient)) { + recipient = previous; + terminalOutput().print( + " ERROR: contact storage is unavailable; reboot and retry\r\n"); + return; + } printTerminalPath(recipient); } @@ -5510,6 +6170,12 @@ void MyMesh::rememberTerminalAck(ContactInfo& recipient, const char* text, } void MyMesh::importTerminalCard(char* command) { + if (_store->hasIncompleteContactLoad()) { + terminalOutput().print( + " ERROR: contact storage is unavailable; reboot and retry\r\n"); + return; + } + while (*command == ' ') command++; if (strncmp(command, "meshcore://", 11) != 0) { terminalOutput().print(" ERROR: invalid card format\r\n"); @@ -6147,9 +6813,15 @@ void MyMesh::handleTerminalCommand(char* command) { if (recipient == NULL) { terminalOutput().print(" ERROR: no recipient selected\r\n"); } else { + const uint8_t previous_out_path_len = recipient->out_path_len; resetPathTo(*recipient); - scheduleContactWrite(*recipient); - terminalOutput().print(" Done.\r\n"); + if (scheduleContactWrite(*recipient)) { + terminalOutput().print(" Done.\r\n"); + } else { + recipient->out_path_len = previous_out_path_len; + terminalOutput().print( + " ERROR: contact storage is unavailable; reboot and retry\r\n"); + } } } else if (strcmp(command, "card") == 0) { mesh::Packet* packet = _prefs.advert_loc_policy == ADVERT_LOC_NONE @@ -6936,22 +7608,62 @@ void MyMesh::checkSerialInterface() { if (len > 0) { handleCmdFrame(len); } else if (_iter_started // check if our ContactsIterator is 'running' + && (_iter_next_frame_at == 0 + || _iter_next_frame_at == _ms->getMillis() + || millisHasNowPassed(_iter_next_frame_at)) && !_serial->isWriteBusy() // don't spam the Serial Interface too quickly! ) { - ContactInfo contact; - if (_iter.hasNext(this, contact)) { - if (contact.lastmod > _iter_filter_since) { // apply the 'since' filter - writeContactRespFrame(RESP_CODE_CONTACT, contact); - if (contact.lastmod > _most_recent_lastmod) { - _most_recent_lastmod = contact.lastmod; // save for the RESP_CODE_END_OF_CONTACTS frame - } + if (_iter_table_revision != getContactTableRevision()) { + // The paced stream walks the live compacting table. If an advert, app + // command, or overwrite changes its membership mid-stream, start a fresh + // snapshot marker before reading another entry. Clients already reset + // their accumulation on CONTACTS_START, so they can never accept a + // skipped, duplicated, or wrong-count list. + _iter = startContactsIterator(); + _iter_start_pending = true; + _iter_contact_pending = false; + _iter_total_count = getNumContacts(); + _iter_table_revision = getContactTableRevision(); + _most_recent_lastmod = 0; + } + + if (_iter_start_pending) { + uint8_t reply[5]; + reply[0] = RESP_CODE_CONTACTS_START; + memcpy(&reply[1], &_iter_total_count, 4); + if (_serial->writeFrame(reply, sizeof(reply)) == sizeof(reply)) { + _iter_start_pending = false; + _iter_next_frame_at = futureMillis(CONTACT_STREAM_FRAME_INTERVAL_MS); } - } else { // EOF - out_frame[0] = RESP_CODE_END_OF_CONTACTS; - memcpy(&out_frame[1], &_most_recent_lastmod, - 4); // include the most recent lastmod, so app can update their 'since' - _serial->writeFrame(out_frame, 5); - stopContactsIterator(); + return; + } + + if (!_iter_contact_pending) { + ContactInfo contact; + if (_iter.hasNext(this, contact)) { + if (contact.lastmod <= _iter_filter_since) return; + _iter_pending_contact = contact; + _iter_contact_pending = true; + } else { // EOF + out_frame[0] = RESP_CODE_END_OF_CONTACTS; + memcpy(&out_frame[1], &_most_recent_lastmod, + 4); // include the most recent lastmod, so app can update their 'since' + if (_serial->writeFrame(out_frame, 5) == 5) { + stopContactsIterator(); + } + return; + } + } + + // Do not advance past a contact until its complete frame has been accepted + // by the selected transport. A transiently full/lost route must not turn a + // successful CONTACTS_START/END transaction into a silently short list. + if (writeContactRespFrame(RESP_CODE_CONTACT, _iter_pending_contact)) { + if (_iter_pending_contact.lastmod > _most_recent_lastmod) { + _most_recent_lastmod = _iter_pending_contact.lastmod; + } + _iter_contact_pending = false; + _iter_next_frame_at = futureMillis(CONTACT_STREAM_FRAME_INTERVAL_MS); } //} else if (!_serial->isWriteBusy()) { // checkConnections(); // TODO - deprecate the 'Connections' stuff diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 005f0524c..0eb86aa17 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -250,8 +250,9 @@ protected: bool shouldAutoAddContactType(uint8_t type) const override; bool shouldOverwriteWhenFull() const override; uint8_t getAutoAddMaxHops() const override; + bool canMutateContacts() const override; void onContactsFull() override; - void onContactOverwrite(const uint8_t* pub_key) override; + bool onContactOverwrite(const ContactInfo& contact) override; bool onContactPathRecv(ContactInfo& from, uint8_t* in_path, uint8_t in_path_len, uint8_t* out_path, uint8_t out_path_len, uint8_t extra_type, uint8_t* extra, uint8_t extra_len) override; void onDiscoveredContact(ContactInfo &contact, bool is_new, uint8_t path_len, const uint8_t* path) override; void onContactPathUpdated(const ContactInfo &contact) override; @@ -325,9 +326,9 @@ private: BaseSerialInterface* route = nullptr); size_t writePendingSerialFrame(const uint8_t frame[], size_t len); void writeDisabledFrame(); - void writeContactRespFrame(uint8_t code, const ContactInfo &contact); + bool writeContactRespFrame(uint8_t code, const ContactInfo &contact); void stopContactsIterator(); - void updateContactFromFrame(ContactInfo &contact, uint32_t& last_mod, const uint8_t *frame, int len); + bool updateContactFromFrame(ContactInfo &contact, uint32_t& last_mod, const uint8_t *frame, int len); void addToOfflineQueue(const uint8_t frame[], int len); int getFromOfflineQueue(uint8_t frame[]); int getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) override { @@ -421,10 +422,17 @@ private: bool flushContactsBeforeReboot(); bool prepareForOtaReboot() override; bool prepareForUiShutdown() override; - void scheduleContactWrite(const ContactInfo& contact); - void scheduleContactWriteAfterRelease(const ContactInfo& contact); + bool scheduleContactWrite(const ContactInfo& contact); + bool scheduleContactWriteAfterRelease(const ContactInfo& contact, + uint16_t& released_slot); + bool restoreContactWriteAfterRelease(const ContactInfo& contact, + uint16_t released_slot); #if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH) void repairInternalExtraFS(Stream& output); +#if defined(MESHCORE_EXTRAFS_HIL) + bool handleExtraFsHilCommand(const char* command, char* reply, + size_t reply_size); +#endif #endif DataStore* _store; @@ -455,10 +463,16 @@ private: AbstractUITask* _ui; ContactsIterator _iter; + ContactInfo _iter_pending_contact; uint32_t _iter_filter_since; + uint32_t _iter_next_frame_at; + uint32_t _iter_total_count; + uint32_t _iter_table_revision; uint32_t _most_recent_lastmod; uint32_t _active_ble_pin; bool _iter_started; + bool _iter_start_pending; + bool _iter_contact_pending; bool _cli_rescue; #ifdef ENABLE_USB_INTERFACE bool _terminal_mode; diff --git a/examples/companion_radio/main.cpp b/examples/companion_radio/main.cpp index 1a78c5fbe..f0d024962 100644 --- a/examples/companion_radio/main.cpp +++ b/examples/companion_radio/main.cpp @@ -156,13 +156,23 @@ static bool isNetworkTerminalActive(); // platform file system #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) #include + #if defined(NRF52_PLATFORM) + #include + #include + #if defined(NRF52840_XXAA) + static const uint32_t INTERNAL_PRIMARY_FS_START = 0x000ED000UL; + #else + static const uint32_t INTERNAL_PRIMARY_FS_START = 0x0006D000UL; + #endif + static const uint32_t INTERNAL_PRIMARY_FS_SIZE = + 7UL * FLASH_NRF52_PAGE_SIZE; + #endif #if defined(QSPIFLASH) #include DataStore store(InternalFS, QSPIFlash, rtc_clock); #else #if defined(EXTRAFS) #include - #include extern "C" uint32_t __flash_arduino_end[]; CustomLFS ExtraFS(0xD4000, 0x19000, 128); DataStore store(InternalFS, ExtraFS, rtc_clock); @@ -2072,7 +2082,46 @@ void setup() { #endif #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) +#if defined(NRF52_PLATFORM) + // InternalFileSystem::begin() auto-formats the entire primary store after a + // mount failure. The base mount plus a full raw scan distinguishes a virgin + // erased device from nonblank/corrupt identity storage. Only proven-erased + // media may be initialized automatically; format is followed by a required + // base remount because Adafruit_LittleFS::format() does not remount an + // already-unmounted instance. + const mesh::storage::InternalSecondaryFsBootResult primary_fs_boot = + mesh::storage::prepareInternalSecondaryFilesystem( + []() -> bool { + return InternalFS.Adafruit_LittleFS::begin(); + }, + []() -> bool { + return mesh::storage::isErasedFlashRange( + INTERNAL_PRIMARY_FS_START, INTERNAL_PRIMARY_FS_SIZE, + [](uint32_t address) -> uint32_t { + return *reinterpret_cast(address); + }); + }, + []() -> bool { + InternalFS.end(); + return InternalFS.format(); + }); + if (primary_fs_boot + == mesh::storage::InternalSecondaryFsBootResult::PreservedNonBlank + || primary_fs_boot + == mesh::storage::InternalSecondaryFsBootResult::InitializationFailed) { + if (primary_fs_boot + == mesh::storage::InternalSecondaryFsBootResult::PreservedNonBlank) { + MESH_DEBUG_PRINTLN( + "InternalFS: mount failed; preserving nonblank primary storage and blocking startup writes"); + } else { + MESH_DEBUG_PRINTLN( + "InternalFS: erased primary storage initialization failed; blocking startup writes"); + } + store.markPrimaryFSUnavailable(); + } +#else InternalFS.begin(); +#endif #if defined(QSPIFLASH) if (!QSPIFlash.begin()) { // debug output might not be available at this point, might be too early. maybe should fall back to InternalFS here? @@ -2086,7 +2135,7 @@ void setup() { NRF_QSPI->TASKS_DEACTIVATE = 1; NRF_QSPI->ENABLE = QSPI_ENABLE_ENABLE_Disabled; #endif - store.disableSecondaryFS(); + store.disableSecondaryFS(true); } else { MESH_DEBUG_PRINTLN("CustomLFS_QSPIFlash: initialized successfully"); } @@ -2134,7 +2183,10 @@ void setup() { } else { MESH_DEBUG_PRINTLN("CustomLFS: blank internal ExtraFS initialization failed; using primary storage"); } - store.disableSecondaryFS(); + const bool secondary_authority_unknown = !extra_fs_geometry_valid + || extra_fs_boot + == mesh::storage::InternalSecondaryFsBootResult::PreservedNonBlank; + store.disableSecondaryFS(secondary_authority_unknown); } #endif #endif diff --git a/src/helpers/ArduinoSerialInterface.cpp b/src/helpers/ArduinoSerialInterface.cpp index 206acbdb3..7f02cf2dd 100644 --- a/src/helpers/ArduinoSerialInterface.cpp +++ b/src/helpers/ArduinoSerialInterface.cpp @@ -226,7 +226,10 @@ size_t ArduinoSerialInterface::writeFrame(const uint8_t src[], size_t len) { if (_passthroughMode) return len; if (_flow_ctl) { if (!isConnected()) { - return len; // nobody is listening, drop instead of filling the TX buffer + // Best-effort pushes may be discarded while nobody is listening. A + // command response or required push must report failure so a multi-frame + // producer can retry it without silently skipping an item. + return mesh::companionFrameRequiresDelivery(src, len) ? 0 : len; } if (!enqueueFrame(src, len)) return 0; serviceTransmit(); diff --git a/src/helpers/BaseChatMesh.cpp b/src/helpers/BaseChatMesh.cpp index 606979caa..1ac3f9024 100644 --- a/src/helpers/BaseChatMesh.cpp +++ b/src/helpers/BaseChatMesh.cpp @@ -129,7 +129,9 @@ ContactInfo* BaseChatMesh::allocateContactSlot(bool transient_only) { #else if (num_contacts < MAX_ANON_CONTACTS+MAX_CONTACTS) { #endif - return &contacts[num_contacts++]; + ContactInfo* allocated = &contacts[num_contacts++]; + contact_table_revision++; + return allocated; } else if (shouldOverwriteWhenFull()) { // Find oldest non-favourite contact by oldest lastmod timestamp for (int i = MAX_ANON_CONTACTS; i < num_contacts; i++) { @@ -140,7 +142,10 @@ ContactInfo* BaseChatMesh::allocateContactSlot(bool transient_only) { } } if (oldest_idx >= 0) { - onContactOverwrite(contacts[oldest_idx].id.pub_key); + if (!onContactOverwrite(contacts[oldest_idx])) { + return NULL; + } + contact_table_revision++; return &contacts[oldest_idx]; } } @@ -171,6 +176,11 @@ void BaseChatMesh::onAdvertRecv(mesh::Packet* packet, const mesh::Identity& id, MESH_DEBUG_PRINTLN("onAdvertRecv: invalid app_data, or name is missing: len=%d", app_data_len); return; } + if (!canMutateContacts()) { + MESH_DEBUG_PRINTLN( + "onAdvertRecv: contact store incomplete; ignoring contact mutation"); + return; + } ContactInfo* from = NULL; for (int i = 0; i < num_contacts; i++) { @@ -287,7 +297,9 @@ void BaseChatMesh::onPeerDataRecv(mesh::Packet* packet, uint8_t type, int sender data[len] = 0; // need to make a C string again, with null terminator if (flags == TXT_TYPE_PLAIN) { - from.lastmod = getRTCClock()->getCurrentTime(); // update last heard time + if (canMutateContacts()) { + from.lastmod = getRTCClock()->getCurrentTime(); // update last heard time + } onMessageRecv(from, packet, sender_timestamp, (const char *) &data[5]); // let UI know int text_len = strlen((char *)&data[5]); @@ -345,10 +357,12 @@ void BaseChatMesh::onPeerDataRecv(mesh::Packet* packet, uint8_t type, int sender } } } else if (flags == TXT_TYPE_SIGNED_PLAIN) { - if (sender_timestamp > from.sync_since) { // make sure 'sync_since' is up-to-date - from.sync_since = sender_timestamp; + if (canMutateContacts()) { + if (sender_timestamp > from.sync_since) { // make sure 'sync_since' is up-to-date + from.sync_since = sender_timestamp; + } + from.lastmod = getRTCClock()->getCurrentTime(); // update last heard time } - from.lastmod = getRTCClock()->getCurrentTime(); // update last heard time onSignedMessageRecv(from, packet, sender_timestamp, &data[5], (const char *) &data[9]); // let UI know uint32_t ack_hash; // calc truncated hash of the message timestamp + text + OUR pub_key, to prove to sender that we got it @@ -410,10 +424,12 @@ bool BaseChatMesh::onPeerPathRecv(mesh::Packet* packet, int sender_idx, const ui bool BaseChatMesh::onContactPathRecv(ContactInfo& from, uint8_t* in_path, uint8_t in_path_len, uint8_t* out_path, uint8_t out_path_len, uint8_t extra_type, uint8_t* extra, uint8_t extra_len) { // NOTE: default impl, we just replace the current 'out_path' regardless, whenever sender sends us a new out_path. // FUTURE: could store multiple out_paths per contact, and try to find which is the 'best'(?) - from.out_path_len = mesh::Packet::copyPath(from.out_path, out_path, out_path_len); // store a copy of path, for sendDirect() - from.lastmod = getRTCClock()->getCurrentTime(); + if (canMutateContacts()) { + from.out_path_len = mesh::Packet::copyPath(from.out_path, out_path, out_path_len); // store a copy of path, for sendDirect() + from.lastmod = getRTCClock()->getCurrentTime(); - onContactPathUpdated(from); + onContactPathUpdated(from); + } if (extra_type == PAYLOAD_TYPE_ACK && extra_len >= 4) { // also got an encoded ACK! @@ -992,6 +1008,38 @@ ContactInfo* BaseChatMesh::lookupContactByPubKey(const uint8_t* pub_key, int pre return NULL; // not found } +ContactInfo* BaseChatMesh::lookupTransientContactByPubKey(const uint8_t* pub_key, int prefix_len) { + for (int i = 0; i < MAX_ANON_CONTACTS; i++) { + auto c = &contacts[i]; + if (memcmp(c->id.pub_key, pub_key, prefix_len) == 0) return c; + } + return NULL; // not found +} + +ContactInfo* BaseChatMesh::lookupPersistentContactByPubKey(const uint8_t* pub_key, int prefix_len) { + for (int i = MAX_ANON_CONTACTS; i < num_contacts; i++) { + auto c = &contacts[i]; + if (memcmp(c->id.pub_key, pub_key, prefix_len) == 0) return c; + } + return NULL; // not found +} + +bool BaseChatMesh::isTransientContact(const ContactInfo& contact) const { + for (int i = 0; i < MAX_ANON_CONTACTS; i++) { + if (&contacts[i] == &contact) return true; + } + return false; +} + +bool BaseChatMesh::clearTransientContact(ContactInfo& contact) { + if (!isTransientContact(contact)) return false; + memset(&contact, 0, sizeof(contact)); +#if defined(NRF52_PLATFORM) + contact.storage_slot = mesh::storage::CONTACT_SLOT_NONE; +#endif + return true; +} + bool BaseChatMesh::addContact(const ContactInfo& contact) { ContactInfo* dest = allocateContactSlot(contact.type == ADV_TYPE_NONE); if (dest) { @@ -1003,8 +1051,10 @@ bool BaseChatMesh::addContact(const ContactInfo& contact) { } bool BaseChatMesh::removeContact(ContactInfo& contact) { - int idx = 0; - while (idx < num_contacts && !contacts[idx].id.matches(contact.id)) { + // removeContact() is for stored contacts only. Match the exact live object so + // duplicate keys cannot remove a transient prefix slot or a different record. + int idx = MAX_ANON_CONTACTS; + while (idx < num_contacts && &contacts[idx] != &contact) { idx++; } if (idx >= num_contacts) return false; // not found @@ -1015,6 +1065,7 @@ bool BaseChatMesh::removeContact(ContactInfo& contact) { contacts[idx] = contacts[idx + 1]; idx++; } + contact_table_revision++; return true; // Success } diff --git a/src/helpers/BaseChatMesh.h b/src/helpers/BaseChatMesh.h index 537e56112..03ed484af 100644 --- a/src/helpers/BaseChatMesh.h +++ b/src/helpers/BaseChatMesh.h @@ -85,6 +85,7 @@ class BaseChatMesh : public mesh::Mesh { int sort_array[MAX_CONTACTS+MAX_ANON_CONTACTS]; #endif int num_contacts; + uint32_t contact_table_revision; int matching_peer_indexes[MAX_SEARCH_RESULTS]; unsigned long txt_send_timeout; #ifdef MAX_GROUP_CHANNELS @@ -107,6 +108,7 @@ protected: sort_array = sort_array_fallback; contact_capacity = CONTACT_PSRAM_FALLBACK_TOTAL_SLOTS; #endif + contact_table_revision = 0; resetContacts(); #ifdef MAX_GROUP_CHANNELS @@ -129,6 +131,7 @@ protected: } #endif num_contacts = MAX_ANON_CONTACTS; // seed the first contacts for anon requests + contact_table_revision++; } void populateContactFromAdvert(ContactInfo& ci, const mesh::Identity& id, const AdvertDataParser& parser, uint32_t timestamp); ContactInfo* allocateContactSlot(bool transient_only=false); // helper to find slot for new contact @@ -139,7 +142,11 @@ protected: virtual void onContactsFull() {}; virtual bool shouldOverwriteWhenFull() const { return false; } virtual uint8_t getAutoAddMaxHops() const { return 0; } // 0 = no limit, 1 = direct (0 hops), N = up to N-1 hops - virtual void onContactOverwrite(const uint8_t* pub_key) {}; + virtual bool canMutateContacts() const { return true; } + // Return false to preserve the exact selected contact when a subclass cannot + // safely retire its associated persistent state. A refusal must not mutate + // the contact or its backing state. + virtual bool onContactOverwrite(const ContactInfo&) { return true; }; virtual void onDiscoveredContact(ContactInfo& contact, bool is_new, uint8_t path_len, const uint8_t* path) = 0; virtual ContactInfo* processAck(const uint8_t *data) = 0; virtual void onContactPathUpdated(const ContactInfo& contact) = 0; @@ -209,10 +216,15 @@ public: void scanRecentContacts(int last_n, ContactVisitor* visitor); ContactInfo* searchContactsByPrefix(const char* name_prefix); ContactInfo* lookupContactByPubKey(const uint8_t* pub_key, int prefix_len); + ContactInfo* lookupTransientContactByPubKey(const uint8_t* pub_key, int prefix_len); + ContactInfo* lookupPersistentContactByPubKey(const uint8_t* pub_key, int prefix_len); + bool isTransientContact(const ContactInfo& contact) const; + bool clearTransientContact(ContactInfo& contact); bool removeContact(ContactInfo& contact); bool addContact(const ContactInfo& contact); int getTotalContactSlots() const { return num_contacts; } int getNumContacts() const { return num_contacts - MAX_ANON_CONTACTS; } // don't include the reserved slots at start + uint32_t getContactTableRevision() const { return contact_table_revision; } bool getContactByIdx(uint32_t idx, ContactInfo& contact); ContactInfo* getContactPtrByIdx(uint32_t idx) { return idx < (uint32_t)num_contacts ? &contacts[idx] : NULL; diff --git a/src/helpers/PersistentStoreFormat.h b/src/helpers/PersistentStoreFormat.h index 8ff516c7d..9089f37db 100644 --- a/src/helpers/PersistentStoreFormat.h +++ b/src/helpers/PersistentStoreFormat.h @@ -22,6 +22,10 @@ static const uint16_t CONTACT_SLOT_NONE = 0xFFFF; static const uint8_t CONTACT_PAGE_VERSION = 1; static const uint8_t CONTACT_PAGE_MAGIC[4] = {'M', 'C', 'P', '4'}; +inline uint32_t contactPageValidSlotMask() { + return (1UL << CONTACTS_PER_PAGE) - 1UL; +} + struct ContactPageHeader { uint8_t page_index; uint32_t occupied; @@ -35,6 +39,22 @@ enum class ContactStoreSource : uint8_t { PAGED, }; +enum class ContactPathState : uint8_t { + ABSENT, + PRESENT, + IO_ERROR, +}; + +// Adafruit_LittleFS::exists() collapses every lfs_stat() error into false. +// Keep the raw result distinguishable so a transient media error can never be +// treated as proof that an authoritative stored source or journal is absent. +inline ContactPathState classifyContactPathStat(int result, + int no_entry_result) { + if (result == 0) return ContactPathState::PRESENT; + if (result == no_entry_result) return ContactPathState::ABSENT; + return ContactPathState::IO_ERROR; +} + // A retained legacy file means migration is still in progress. Its complete // record count is the authoritative prefix boundary; page records at or above // that boundary have already committed. @@ -59,6 +79,12 @@ inline uint16_t legacyContactCountForSize(size_t file_size) { return (uint16_t)(count < capacity ? count : capacity); } +inline bool isValidLegacyContactFileSize(size_t file_size) { + const size_t capacity = CONTACT_PAGE_COUNT * CONTACTS_PER_PAGE; + return file_size % CONTACT_RECORD_SIZE == 0 + && file_size <= capacity * CONTACT_RECORD_SIZE; +} + inline uint8_t legacyMigrationPage(uint16_t legacy_contact_count) { return legacy_contact_count == 0 ? CONTACT_PAGE_COUNT : (uint8_t)((legacy_contact_count - 1) / CONTACTS_PER_PAGE); @@ -112,6 +138,13 @@ inline uint32_t updateCRC32(uint32_t crc, const uint8_t* data, size_t len) { return crc; } +inline bool contactRecordHasData( + const uint8_t record[CONTACT_RECORD_SIZE]) { + uint8_t combined = 0; + for (uint16_t i = 0; i < CONTACT_RECORD_SIZE; i++) combined |= record[i]; + return combined != 0; +} + inline void encodeContactPageHeader(uint8_t dest[CONTACT_PAGE_HEADER_SIZE], const ContactPageHeader& header) { memcpy(dest, CONTACT_PAGE_MAGIC, sizeof(CONTACT_PAGE_MAGIC)); @@ -126,7 +159,6 @@ inline void encodeContactPageHeader(uint8_t dest[CONTACT_PAGE_HEADER_SIZE], inline bool decodeContactPageHeader(const uint8_t src[CONTACT_PAGE_HEADER_SIZE], uint8_t expected_page, ContactPageHeader& header) { - const uint32_t valid_slots = (1UL << CONTACTS_PER_PAGE) - 1UL; if (memcmp(src, CONTACT_PAGE_MAGIC, sizeof(CONTACT_PAGE_MAGIC)) != 0 || src[4] != CONTACT_PAGE_VERSION || src[5] != expected_page || readLE16(&src[6]) != CONTACT_RECORD_SIZE) { @@ -137,7 +169,10 @@ inline bool decodeContactPageHeader(const uint8_t src[CONTACT_PAGE_HEADER_SIZE], header.occupied = readLE32(&src[8]); header.generation = readLE32(&src[12]); header.payload_crc = readLE32(&src[16]); - return (header.occupied & ~valid_slots) == 0; + // Occupancy is repairable from the CRC-protected record payload. Keep the + // header parseable even when an unused high bit was upset so load can retain + // the page's contacts and rewrite the corrected mask. + return true; } class DirtyPageSet { diff --git a/src/helpers/nrf52/InternalSecondaryFsRepair.h b/src/helpers/nrf52/InternalSecondaryFsRepair.h index f9e04a71f..3080e5895 100644 --- a/src/helpers/nrf52/InternalSecondaryFsRepair.h +++ b/src/helpers/nrf52/InternalSecondaryFsRepair.h @@ -20,6 +20,21 @@ inline bool isInternalExtraFsReservedByApplication(uint32_t application_end) { return application_end <= INTERNAL_EXTRAFS_START; } +// Read flash through an injected word reader so the erased-media decision can +// be covered by native tests. Refuse malformed/empty ranges: only a complete, +// word-aligned scan can authorize a boot-time format. +template +bool isErasedFlashRange(uint32_t start, uint32_t size, ReadWord read_word) { + if (size == 0 || (start % sizeof(uint32_t)) != 0 + || (size % sizeof(uint32_t)) != 0) { + return false; + } + for (uint32_t offset = 0; offset < size; offset += sizeof(uint32_t)) { + if (read_word(start + offset) != 0xFFFFFFFFUL) return false; + } + return true; +} + enum class InternalSecondaryFsBootResult { Mounted, InitializedBlank, diff --git a/test/test_companion_contact_persistence_contract.py b/test/test_companion_contact_persistence_contract.py new file mode 100644 index 000000000..0e347cf2e --- /dev/null +++ b/test/test_companion_contact_persistence_contract.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Static contracts for failure-atomic Companion contact mutations.""" + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +def source(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def function_body(text: str, signature: str) -> str: + start = text.index(signature) + opening = text.index("{", start) + depth = 0 + for position in range(opening, len(text)): + if text[position] == "{": + depth += 1 + elif text[position] == "}": + depth -= 1 + if depth == 0: + return text[opening + 1 : position] + raise AssertionError(f"unterminated function: {signature}") + + +def command_section(handler: str, command: str, next_command: str) -> str: + start = handler.index(f"cmd_frame[0] == {command}") + end = handler.index(f"cmd_frame[0] == {next_command}", start) + return handler[start:end] + + +class CompanionContactPersistenceContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.mesh = source("examples/companion_radio/MyMesh.cpp") + cls.mesh_header = source("examples/companion_radio/MyMesh.h") + cls.data_store = source("examples/companion_radio/DataStore.cpp") + cls.data_store_header = source("examples/companion_radio/DataStore.h") + cls.base = source("src/helpers/BaseChatMesh.cpp") + cls.base_header = source("src/helpers/BaseChatMesh.h") + cls.handler = function_body(cls.mesh, "void MyMesh::handleCmdFrame(") + + def test_scheduling_results_are_observable(self): + self.assertIn( + "bool scheduleContactWrite(const ContactInfo& contact);", + self.mesh_header, + ) + self.assertIn( + "bool scheduleContactWriteAfterRelease(const ContactInfo& contact,", + self.mesh_header, + ) + self.assertIn( + "bool restoreContactWriteAfterRelease(const ContactInfo& contact,", + self.mesh_header, + ) + self.assertIn( + "bool restoreContactSlot(const ContactInfo& contact, uint16_t slot);", + self.data_store_header, + ) + + dirty = function_body( + self.mesh, + "bool MyMesh::scheduleContactWrite(const ContactInfo& contact)", + ) + transient_guard = dirty.index( + "contact.type == ADV_TYPE_NONE || isTransientContact(contact)" + ) + mark = dirty.index("if (!_store->markContactDirty(contact))") + self.assertLess(transient_guard, mark) + self.assertIn("if (!_store->markContactDirty(contact))", dirty) + self.assertIn("return false;", dirty) + self.assertIn("scheduleLazyPersistenceMutation(", dirty) + self.assertTrue(dirty.rstrip().endswith("return true;")) + + release = function_body( + self.mesh, + "bool MyMesh::scheduleContactWriteAfterRelease(" + "const ContactInfo& contact,", + ) + self.assertIn("released_slot = contact.storage_slot", release) + self.assertIn("if (!_store->releaseContact(contact))", release) + self.assertIn("return false;", release) + self.assertIn("scheduleLazyPersistenceMutation(", release) + self.assertTrue(release.rstrip().endswith("return true;")) + + restore = function_body( + self.mesh, + "bool MyMesh::restoreContactWriteAfterRelease(" + "const ContactInfo& contact,", + ) + self.assertIn( + "if (!_store->restoreContactSlot(contact, released_slot))", + restore, + ) + + def test_release_rollback_reserves_the_exact_original_slot(self): + restore = function_body( + self.data_store, + "bool DataStore::restoreContactSlot(const ContactInfo& contact,", + ) + reserve = restore.index("_contact_slots.reserve(slot)") + mark = restore.index("_dirty_contact_pages.mark(", reserve) + assign = restore.index("contact.storage_slot = slot", mark) + self.assertLess(reserve, mark) + self.assertLess(mark, assign) + self.assertNotIn("_contact_slots.allocate()", restore) + + def test_page_load_uses_one_verified_payload_snapshot(self): + load = function_body( + self.data_store, + "bool DataStore::loadContactPages(DataStoreHost* host,", + ) + self.assertIn( + "malloc(mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE)", load + ) + self.assertIn( + "file.read(payload, mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE)", + load, + ) + self.assertIn( + "mesh::storage::updateCRC32(\n" + " 0xFFFFFFFFUL, payload,", + load, + ) + parse_loop = load.index( + "for (uint8_t index = 0; " + "index < mesh::storage::CONTACTS_PER_PAGE; index++)" + ) + self.assertNotIn("file.seek", load[parse_loop:]) + self.assertNotIn("file.read", load[parse_loop:]) + quarantine = load.index("auto quarantineUnreadPage") + first_read_failure = load.index("if (!file)", quarantine) + first_quarantine = load.index( + "quarantineUnreadPage(page)", first_read_failure + ) + self.assertLess(first_read_failure, first_quarantine) + self.assertGreaterEqual(load.count("quarantineUnreadPage(page)"), 4) + + def test_reset_path_rolls_back_before_reporting_persistence_error(self): + section = command_section( + self.handler, "CMD_RESET_PATH", "CMD_ADD_UPDATE_CONTACT" + ) + self.assertIn("lookupPersistentContactByPubKey", section) + snapshot = section.index("previous_out_path_len") + mutation = section.index("recipient->out_path_len = OUT_PATH_UNKNOWN") + schedule = section.index("if (scheduleContactWrite(*recipient))") + rollback = section.index( + "recipient->out_path_len = previous_out_path_len", schedule + ) + error = section.index("writeErrFrame(ERR_CODE_FILE_IO_ERROR)", rollback) + ok = section.index("writeOKFrame()", schedule) + clear_transient = section.index("clearTransientContact(*transient)", schedule) + self.assertLess(snapshot, mutation) + self.assertLess(mutation, schedule) + self.assertLess(schedule, clear_transient) + self.assertLess(clear_transient, ok) + self.assertLess(schedule, ok) + self.assertLess(ok, rollback) + self.assertLess(rollback, error) + + def test_add_update_frame_is_validated_before_contact_lookup(self): + section = command_section( + self.handler, "CMD_ADD_UPDATE_CONTACT", "CMD_REMOVE_CONTACT" + ) + short = section.index("len < CONTACT_UPDATE_FRAME_MIN_LEN") + illegal = section.index("writeErrFrame(ERR_CODE_ILLEGAL_ARG)", short) + lookup = section.index("lookupPersistentContactByPubKey", illegal) + self.assertLess(short, illegal) + self.assertLess(illegal, lookup) + self.assertNotIn("CMD_ADD_UPDATE_CONTACT\n && len", section) + + parser = function_body( + self.mesh, "bool MyMesh::updateContactFromFrame(" + ) + self.assertIn("len < CONTACT_UPDATE_FRAME_MIN_LEN", parser) + self.assertIn("frame[0] != CMD_ADD_UPDATE_CONTACT", parser) + self.assertIn("!isPersistentContactType(type)", parser) + self.assertIn("out_path_len != OUT_PATH_UNKNOWN", parser) + self.assertIn("!mesh::Packet::isValidPathLen(out_path_len)", parser) + self.assertIn("contact.name[sizeof(contact.name) - 1] = 0", parser) + + valid_types = function_body( + self.mesh, "static bool isPersistentContactType(uint8_t type)" + ) + for contact_type in ( + "ADV_TYPE_CHAT", + "ADV_TYPE_REPEATER", + "ADV_TYPE_ROOM", + "ADV_TYPE_SENSOR", + ): + self.assertIn(contact_type, valid_types) + + def test_partially_present_optional_contact_fields_are_rejected(self): + parser = function_body( + self.mesh, "bool MyMesh::updateContactFromFrame(" + ) + self.assertIn("CONTACT_UPDATE_FRAME_GPS_LEN", parser) + self.assertIn("CONTACT_UPDATE_FRAME_LASTMOD_LEN", parser) + self.assertIn( + "len > CONTACT_UPDATE_FRAME_MIN_LEN\n" + " && len < CONTACT_UPDATE_FRAME_GPS_LEN", + parser, + ) + self.assertIn( + "len > CONTACT_UPDATE_FRAME_GPS_LEN\n" + " && len < CONTACT_UPDATE_FRAME_LASTMOD_LEN", + parser, + ) + + def test_update_and_add_ack_only_after_scheduling(self): + section = command_section( + self.handler, "CMD_ADD_UPDATE_CONTACT", "CMD_REMOVE_CONTACT" + ) + + update_snapshot = section.index("const ContactInfo previous = *recipient") + update_schedule = section.index("if (scheduleContactWrite(*recipient))") + update_ok = section.index("writeOKFrame()", update_schedule) + update_rollback = section.index("*recipient = previous", update_ok) + update_error = section.index( + "writeErrFrame(ERR_CODE_FILE_IO_ERROR)", update_rollback + ) + self.assertLess(update_snapshot, update_schedule) + self.assertLess(update_schedule, update_ok) + self.assertLess(update_ok, update_rollback) + self.assertLess(update_rollback, update_error) + + promotion = section.index("Promotion must free the exact transient prefix slot") + add_section = section[promotion:] + clear = add_section.index("clearTransientContact(*transient)") + add = add_section.index("addContact(candidate)") + add_schedule = section.index( + "added != NULL && scheduleContactWrite(*added)", update_error + ) + add_ok = section.index("writeOKFrame()", add_schedule) + add_rollback = section.index("removeContact(*added)", add_ok) + add_error = section.index( + "writeErrFrame(ERR_CODE_FILE_IO_ERROR)", add_rollback + ) + self.assertLess(clear, add) + self.assertLess(add_schedule, add_ok) + self.assertLess(add_ok, add_rollback) + self.assertLess(add_rollback, add_error) + self.assertIn("if (transient != NULL) *transient = previous_transient", add_section) + + def test_transient_prefix_is_never_compacted(self): + transient_lookup = function_body( + self.base, + "ContactInfo* BaseChatMesh::lookupTransientContactByPubKey(", + ) + persistent_lookup = function_body( + self.base, + "ContactInfo* BaseChatMesh::lookupPersistentContactByPubKey(", + ) + clear = function_body( + self.base, "bool BaseChatMesh::clearTransientContact(" + ) + remove = function_body(self.base, "bool BaseChatMesh::removeContact(") + + self.assertIn("int i = 0; i < MAX_ANON_CONTACTS", transient_lookup) + self.assertIn( + "int i = MAX_ANON_CONTACTS; i < num_contacts", persistent_lookup + ) + self.assertIn("if (!isTransientContact(contact)) return false", clear) + self.assertNotIn("num_contacts", clear) + self.assertIn("int idx = MAX_ANON_CONTACTS", remove) + self.assertIn("&contacts[idx] != &contact", remove) + self.assertNotIn("id.matches", remove) + + def test_remove_reserves_persistent_deletion_before_live_mutation(self): + section = command_section( + self.handler, "CMD_REMOVE_CONTACT", "CMD_SHARE_CONTACT" + ) + self.assertIn("lookupPersistentContactByPubKey", section) + schedule = section.index("!scheduleContactWriteAfterRelease(") + error = section.index("writeErrFrame(ERR_CODE_FILE_IO_ERROR)", schedule) + blob = section.index("!_store->deleteBlobByKey", error) + rollback = section.index( + "restoreContactWriteAfterRelease(*recipient, released_slot)", blob + ) + remove = section.index("removeContact(*recipient)", rollback) + clear = section.index("clearTransientContact(*transient)", remove) + ok = section.index("writeOKFrame()", clear) + self.assertLess(schedule, error) + self.assertLess(error, blob) + self.assertLess(blob, rollback) + self.assertLess(rollback, remove) + self.assertLess(remove, clear) + self.assertLess(clear, ok) + + def test_incomplete_contact_load_is_rejected_before_mutation_lookup(self): + for command, next_command in ( + ("CMD_RESET_PATH", "CMD_ADD_UPDATE_CONTACT"), + ("CMD_ADD_UPDATE_CONTACT", "CMD_REMOVE_CONTACT"), + ("CMD_REMOVE_CONTACT", "CMD_SHARE_CONTACT"), + ): + section = command_section(self.handler, command, next_command) + gate = section.index("_store->hasIncompleteContactLoad()") + error = section.index("writeErrFrame(ERR_CODE_FILE_IO_ERROR)", gate) + lookup = section.index("lookupPersistentContactByPubKey", error) + self.assertLess(gate, error) + self.assertLess(error, lookup) + + def test_failed_persistent_release_vetoes_contact_overwrite(self): + self.assertIn( + "virtual bool onContactOverwrite(const ContactInfo&)", + self.base_header, + ) + allocator_start = self.base.index( + "ContactInfo* BaseChatMesh::allocateContactSlot(" + ) + allocator_end = self.base.index( + "void BaseChatMesh::populateContactFromAdvert(", allocator_start + ) + allocator = self.base[allocator_start:allocator_end] + self.assertIn( + "if (!onContactOverwrite(contacts[oldest_idx]))", + allocator, + ) + + overwrite = function_body( + self.mesh, "bool MyMesh::onContactOverwrite(const ContactInfo& contact)" + ) + release = overwrite.index("!scheduleContactWriteAfterRelease(") + refused = overwrite.index("return false;", release) + delete = overwrite.index("!_store->deleteBlobByKey", refused) + rollback = overwrite.index( + "restoreContactWriteAfterRelease(contact, released_slot)", delete + ) + notify = overwrite.index("PUSH_CODE_CONTACT_DELETED", delete) + self.assertLess(release, refused) + self.assertLess(refused, delete) + self.assertLess(delete, rollback) + self.assertLess(rollback, notify) + self.assertLess(delete, notify) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_companion_contact_sleep_contract.py b/test/test_companion_contact_sleep_contract.py index c3d28b1f0..d1557bbfa 100644 --- a/test/test_companion_contact_sleep_contract.py +++ b/test/test_companion_contact_sleep_contract.py @@ -62,9 +62,8 @@ class CompanionContactSleepContractTest(unittest.TestCase): mesh = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text() for signature in ( - "void MyMesh::scheduleContactWrite(const ContactInfo& contact)", - "void MyMesh::scheduleContactWriteAfterRelease(" - "const ContactInfo& contact)", + "bool MyMesh::scheduleContactWrite(const ContactInfo& contact)", + "bool MyMesh::scheduleContactWriteAfterRelease(", ): body = function_body(mesh, signature) self.assertIn("mesh::scheduleLazyPersistenceMutation(", body) diff --git a/test/test_companion_contact_stream_contract.py b/test/test_companion_contact_stream_contract.py new file mode 100644 index 000000000..e6c243bb0 --- /dev/null +++ b/test/test_companion_contact_stream_contract.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Static contracts for lossless Companion contact-list streaming.""" + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +def source(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def function_body(text: str, signature: str) -> str: + start = text.index(signature) + brace = text.index("{", start) + depth = 0 + for index in range(brace, len(text)): + if text[index] == "{": + depth += 1 + elif text[index] == "}": + depth -= 1 + if depth == 0: + return text[brace + 1 : index] + raise AssertionError(f"unterminated function {signature}") + + +class ContactStreamContractTests(unittest.TestCase): + def test_contact_frame_write_result_is_observable(self): + header = source("examples/companion_radio/MyMesh.h") + impl = source("examples/companion_radio/MyMesh.cpp") + self.assertIn( + "bool writeContactRespFrame(uint8_t code, const ContactInfo &contact);", + header, + ) + body = function_body( + impl, + "bool MyMesh::writeContactRespFrame(uint8_t code, const ContactInfo &contact)", + ) + self.assertRegex( + body, + r"return\s+_serial->writeFrame\(out_frame,\s*i\)\s*==\s*\(size_t\)i\s*;", + ) + + def test_iterator_retries_start_contact_and_end_frames(self): + impl = source("examples/companion_radio/MyMesh.cpp") + body = function_body(impl, "void MyMesh::checkSerialInterface()") + self.assertIn("if (_iter_start_pending)", body) + self.assertIn("_iter_pending_contact = contact;", body) + self.assertIn("_iter_contact_pending = true;", body) + self.assertRegex( + body, + r"if\s*\(writeContactRespFrame\(RESP_CODE_CONTACT,\s*" + r"_iter_pending_contact\)\)", + ) + self.assertRegex( + body, + r"if\s*\(_serial->writeFrame\(out_frame,\s*5\)\s*==\s*5\)\s*\{\s*" + r"stopContactsIterator\(\)\s*;", + ) + + # The cached contact is cleared only inside the successful-write arm. + success = body.index( + "if (writeContactRespFrame(RESP_CODE_CONTACT, _iter_pending_contact))" + ) + clear = body.index("_iter_contact_pending = false;", success) + self.assertGreater(clear, success) + + def test_contact_stream_is_paced_for_slow_usb_consumers(self): + impl = source("examples/companion_radio/MyMesh.cpp") + self.assertIn("#define CONTACT_STREAM_FRAME_INTERVAL_MS 5", impl) + body = function_body(impl, "void MyMesh::checkSerialInterface()") + self.assertIn("millisHasNowPassed(_iter_next_frame_at)", body) + self.assertGreaterEqual( + body.count( + "_iter_next_frame_at = " + "futureMillis(CONTACT_STREAM_FRAME_INTERVAL_MS);" + ), + 2, + ) + + def test_structural_contact_change_restarts_the_stream_snapshot(self): + impl = source("examples/companion_radio/MyMesh.cpp") + base = source("src/helpers/BaseChatMesh.cpp") + base_header = source("src/helpers/BaseChatMesh.h") + body = function_body(impl, "void MyMesh::checkSerialInterface()") + + self.assertIn("uint32_t getContactTableRevision() const", base_header) + self.assertGreaterEqual(base.count("contact_table_revision++;"), 2) + self.assertIn( + "_iter_table_revision != getContactTableRevision()", body + ) + restart = body.index( + "_iter_table_revision != getContactTableRevision()" + ) + marker = body.index("if (_iter_start_pending)", restart) + for statement in ( + "_iter = startContactsIterator();", + "_iter_start_pending = true;", + "_iter_contact_pending = false;", + "_iter_total_count = getNumContacts();", + "_iter_table_revision = getContactTableRevision();", + ): + self.assertLess(body.index(statement, restart), marker) + + def test_disconnected_flow_control_reports_required_frame_failure(self): + impl = source("src/helpers/ArduinoSerialInterface.cpp") + body = function_body( + impl, + "size_t ArduinoSerialInterface::writeFrame(const uint8_t src[], size_t len)", + ) + self.assertIn( + "return mesh::companionFrameRequiresDelivery(src, len) ? 0 : len;", + body, + ) + self.assertNotIn( + "return len; // nobody is listening, drop instead of filling the TX buffer", + body, + ) + + def test_hil_requires_advertised_contact_count(self): + harness = source("tools/hil/t1000e_extrafs_stress.py") + start = harness.index(" async def _contact_snapshot_once(") + body = harness[start : harness.index(" async def channels(self)", start)] + self.assertIn('getattr(reader, "contact_nb", None)', body) + self.assertIn("if len(snapshot) == expected", body) + self.assertIn("incomplete contact enumeration", body) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_companion_extrafs_hil_guard.py b/test/test_companion_extrafs_hil_guard.py new file mode 100644 index 000000000..894af3e79 --- /dev/null +++ b/test/test_companion_extrafs_hil_guard.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Static security contract for the destructive Companion ExtraFS HIL hook.""" + +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_PATH = ROOT / "examples" / "companion_radio" / "MyMesh.cpp" +HEADER_PATH = ROOT / "examples" / "companion_radio" / "MyMesh.h" +HARNESS_PATH = ROOT / "tools" / "hil" / "t1000e_extrafs_stress.py" + +INTERNAL_EXTRAFS_GUARD = ( + "defined(NRF52_PLATFORM)&&defined(EXTRAFS)&&!defined(QSPIFLASH)" +) +HIL_GUARD = "defined(MESHCORE_EXTRAFS_HIL)" +INVALID_TARGET_GUARD = ( + "defined(MESHCORE_EXTRAFS_HIL)&&" + "(!defined(NRF52_PLATFORM)||!defined(EXTRAFS)||defined(QSPIFLASH))" +) + + +def function_body(source: str, signature: str) -> str: + start = source.index(signature) + opening = source.index("{", start) + depth = 0 + for position in range(opening, len(source)): + if source[position] == "{": + depth += 1 + elif source[position] == "}": + depth -= 1 + if depth == 0: + return source[opening + 1 : position] + raise AssertionError(f"unterminated function: {signature}") + + +def canonical_guard(expression: str) -> str: + return re.sub(r"\s+", "", expression.replace("\\", "")) + + +def active_preprocessor_guards(source: str, offset: int) -> list[str]: + """Return the active #if expressions at the source line containing offset.""" + target_line = source[:offset].count("\n") + lines = source.splitlines() + guards: list[str] = [] + line_number = 0 + + while line_number <= target_line: + logical = lines[line_number] + while logical.rstrip().endswith("\\"): + line_number += 1 + logical = logical.rstrip()[:-1] + lines[line_number] + + directive = logical.strip() + if directive.startswith("#if "): + guards.append(canonical_guard(directive[4:])) + elif directive.startswith("#ifdef "): + guards.append(f"defined({directive[7:].strip()})") + elif directive.startswith("#ifndef "): + guards.append(f"!defined({directive[8:].strip()})") + elif directive.startswith("#elif "): + if not guards: + raise AssertionError("#elif without #if") + guards[-1] = canonical_guard(directive[6:]) + elif directive == "#else": + if not guards: + raise AssertionError("#else without #if") + guards[-1] = f"!({guards[-1]})" + elif directive == "#endif": + if not guards: + raise AssertionError("#endif without #if") + guards.pop() + + line_number += 1 + + return guards + + +class CompanionExtraFsHilGuardTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.source = SOURCE_PATH.read_text(encoding="utf-8") + cls.header = HEADER_PATH.read_text(encoding="utf-8") + cls.harness = HARNESS_PATH.read_text(encoding="utf-8") + + def test_enabling_hil_on_any_other_storage_target_is_a_build_error(self): + error_text = ( + '#error "MESHCORE_EXTRAFS_HIL requires an nRF52 internal ' + 'ExtraFS build"' + ) + error_at = self.source.index(error_text) + self.assertIn( + INVALID_TARGET_GUARD, + active_preprocessor_guards(self.source, error_at), + ) + + def test_hil_api_exists_only_inside_both_compile_time_guards(self): + declaration_at = self.header.index( + "bool handleExtraFsHilCommand(const char* command" + ) + definition_at = self.source.index( + "bool MyMesh::handleExtraFsHilCommand(const char* command" + ) + + for text, location in ( + (self.header, declaration_at), + (self.source, definition_at), + ): + active = active_preprocessor_guards(text, location) + self.assertIn(INTERNAL_EXTRAFS_GUARD, active) + self.assertIn(HIL_GUARD, active) + + def test_only_direct_companion_run_cli_intercepts_the_hil_command(self): + # Exclude the qualified MyMesh:: definition: exactly one invocation is + # allowed in the whole implementation. + invocations = list( + re.finditer(r"(?= 32", corrupt) + self.assertIn("_store->hasPendingContactWrites()", corrupt) + self.assertIn("const uint32_t byte_offset = 8 + slot / 8;", corrupt) + self.assertIn("update.flush();", corrupt) + self.assertIn("update.close();", corrupt) + self.assertIn("const bool changed_on_flash", corrupt) + self.assertNotIn("CONTACT_PAGE_HEADER_SIZE)", corrupt) + + def test_page_mask_reads_the_persisted_header(self): + body = function_body( + self.source, + "bool MyMesh::handleExtraFsHilCommand(const char* command", + ) + start = body.index( + 'static const char* const page_mask_prefix = "page-mask ";' + ) + end = body.index( + 'static const char* const corrupt_occupied_prefix', start + ) + page_mask = body[start:end] + + self.assertIn("page_file.read(raw_header", page_mask) + self.assertIn("decodeContactPageHeader(", page_mask) + self.assertIn('occupied=%08lx', page_mask) + + def test_advert_cache_fault_hooks_are_direct_only_and_confirmed(self): + body = function_body( + self.source, + "bool MyMesh::handleExtraFsHilCommand(const char* command", + ) + for command in ( + '"contact-slot "', + '"seed-advert "', + '"clear-advert "', + '"fail-read-page "', + '"fail-stat-page "', + ): + self.assertIn(command, body) + self.assertGreaterEqual(body.count('strcmp(encoded + 14, confirm)'), 2) + self.assertIn("_store->putBlobByKey(", body) + self.assertIn("_store->deleteBlobByKey(", body) + self.assertIn("_store->restoreContactSlot", self.source) + self.assertIn("_store->hasPendingContactWrites()", body) + + def test_read_and_stat_faults_share_one_exclusive_marker(self): + body = function_body( + self.source, + "bool MyMesh::handleExtraFsHilCommand(const char* command", + ) + arm = function_body(body, "auto armContactPageFailure =") + + self.assertEqual(body.count('"/__hil.readfail"'), 1) + self.assertIn("contact_stat_failure_flag = 0x80", body) + self.assertIn("fs->exists(contact_failure_marker)", arm) + self.assertIn("return -1;", arm) + self.assertIn("marker.write(&encoded_failure, 1)", arm) + self.assertIn("verify.read() == encoded_failure", arm) + self.assertGreaterEqual( + body.count("armContactPageFailure("), 2 + ) # both command modes invoke the shared writer + self.assertIn( + 'strcmp(end, " CONFIRM") != 0', + body[body.index('"fail-stat-page "'):], + ) + self.assertGreaterEqual( + body.count('"HIL contact failure already armed"'), 2 + ) + + def test_destructive_contact_faults_reject_incomplete_loads(self): + body = function_body( + self.source, + "bool MyMesh::handleExtraFsHilCommand(const char* command", + ) + sections = ( + ('static const char* const fail_stat_prefix = "fail-stat-page ";', + 'static const char* const fail_read_prefix = "fail-read-page ";'), + ('static const char* const fail_read_prefix = "fail-read-page ";', + 'static const char* const fill_prefix = "fill ";'), + ('static const char* const corrupt_occupied_prefix = "corrupt-occupied ";', + "if (strncmp(action, corrupt_prefix"), + ("if (strncmp(action, corrupt_prefix", + 'if (strcmp(action, "unsafe-reset CONFIRM")'), + ) + for start_marker, end_marker in sections: + start = body.index(start_marker) + end = body.index(end_marker, start + len(start_marker)) + command = body[start:end] + self.assertIn("_store->hasPendingContactWrites()", command) + self.assertIn("_store->hasIncompleteContactLoad()", command) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_companion_settings_persistence_contract.py b/test/test_companion_settings_persistence_contract.py new file mode 100644 index 000000000..877d5832f --- /dev/null +++ b/test/test_companion_settings_persistence_contract.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Static contracts for fail-closed Companion preferences and channels.""" + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +def source(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def function_body(text: str, signature: str) -> str: + start = text.index(signature) + opening = text.index("{", start) + depth = 0 + for position in range(opening, len(text)): + if text[position] == "{": + depth += 1 + elif text[position] == "}": + depth -= 1 + if depth == 0: + return text[opening + 1 : position] + raise AssertionError(f"unterminated function: {signature}") + + +class CompanionSettingsPersistenceContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.store = source("examples/companion_radio/DataStore.cpp") + cls.store_header = source("examples/companion_radio/DataStore.h") + cls.mesh = source("examples/companion_radio/MyMesh.cpp") + + def test_preferences_load_has_an_observable_fail_closed_result(self): + self.assertIn("bool loadPrefs(CompanionNodePrefs& prefs", self.store_header) + self.assertIn("bool loadPrefsInt(const char *filename", self.store_header) + self.assertIn("bool _prefs_load_incomplete = false;", self.store_header) + + save = function_body( + self.store, + "bool DataStore::savePrefs(const CompanionNodePrefs& _prefs", + ) + self.assertRegex( + save, + r"if\s*\([^)]*_prefs_load_incomplete[^)]*\)\s*return false;", + ) + + def test_preferences_source_discovery_does_not_treat_stat_error_as_absence(self): + load = function_body( + self.store, + "bool DataStore::loadPrefs(CompanionNodePrefs& prefs", + ) + self.assertIn( + 'contactPathPresence(_fs, "/new_prefs",', + load, + ) + self.assertIn( + 'contactPathPresence(_fs, "/node_prefs",', + load, + ) + nrf_branch = load[: load.index("#else")] + self.assertNotIn('_fs->exists("/new_prefs")', nrf_branch) + self.assertNotIn('_fs->exists("/node_prefs")', nrf_branch) + self.assertIn("_prefs_load_incomplete = true;", load) + self.assertIn("return false;", load) + self.assertIn("return true;", load) + + def test_preferences_load_is_staged_and_checks_mandatory_reads(self): + load = function_body( + self.store, + "bool DataStore::loadPrefsInt(const char *filename", + ) + + # A failed read must not leave a partly decoded structure in live use. + self.assertRegex( + load, + r"CompanionNodePrefs\s+\w+\s*=\s*_prefs\s*;", + ) + self.assertRegex(load, r"double\s+\w+\s*=\s*node_lat\s*;") + self.assertRegex(load, r"double\s+\w+\s*=\s*node_lon\s*;") + + # All fields in the original 84-byte base image are mandatory. The loader + # must propagate a short underlying read instead of interpreting it as + # defaults, and an unrecognized partial optional tail must also fail. + self.assertRegex( + load, + r"file\.read\([^;]+\)\s*!=\s*size", + ) + self.assertGreaterEqual(load.count("readField("), 22) + self.assertGreaterEqual(load.count("readOptionalField("), 26) + self.assertIn("success = success && file.available() == 0;", load) + failure_at = load.index("if (!success) return false;") + commit_at = load.index("_prefs = loaded_prefs;", failure_at) + self.assertLess(failure_at, commit_at) + self.assertIn("return false;", load) + self.assertIn("return true;", load) + + def test_startup_never_rewrites_preferences_after_failed_load(self): + begin = function_body(self.mesh, "void MyMesh::begin(bool has_display") + load_at = begin.index("prefs_ready") + self.assertIn("_store->loadPrefs(", begin[load_at:]) + migration_at = begin.index("migrateCompanionPowerSavingDefault", load_at) + save_at = begin.index("_store->savePrefs(", migration_at) + guarded_region = begin[migration_at:save_at] + + self.assertRegex( + guarded_region, + r"if\s*\(prefs_ready\s*&&\s*\(", + ) + + def test_channel_source_discovery_and_open_failures_latch_quarantine(self): + load = function_body(self.store, "void DataStore::loadChannels(") + presence_at = load.index( + 'contactPathPresence(contacts_fs, "/channels2",' + ) + open_at = load.index('openRead(_getContactsChannelsFS(), "/channels2")') + self.assertLess(presence_at, open_at) + + # Stat and open errors are distinct from a cleanly absent file, and + # both must enter the same write-veto state used by saveChannels(). + self.assertGreaterEqual(load.count("_contact_load_incomplete = true;"), 2) + self.assertIn("if (!channels_exist) return;", load) + + def test_channel_file_requires_complete_fixed_size_records(self): + load = function_body(self.store, "void DataStore::loadChannels(") + self.assertRegex( + load, + r"CHANNEL_RECORD_SIZE\s*=\s*4\s*\+\s*32\s*\+\s*32", + ) + self.assertRegex( + load, + r"channels_size\s*%\s*CHANNEL_RECORD_SIZE\)\s*!=\s*0", + ) + + # Once the exact record count is known, every selected record must be + # read fully; short reads and host-capacity refusal are errors, not EOF. + self.assertRegex(load, r"file\.read\([^;]+\)\s*==") + self.assertIn("if (!success)", load) + self.assertRegex(load, r"if\s*\(!host->onChannelLoaded\(") + self.assertGreaterEqual(load.count("_contact_load_incomplete = true;"), 4) + + save = function_body(self.store, "bool DataStore::saveChannels(") + self.assertIn("if (_contact_load_incomplete) return false;", save) + + def test_failed_legacy_preferences_recovery_latches_preferences(self): + migrate = function_body(self.store, "bool DataStore::migrateToSecondaryFS()") + primary = function_body( + migrate, + "auto migratePrimarySources = [this, ©, &filesEqual]()", + ) + self.assertIn( + 'const bool is_preferences = strcmp(path, "/new_prefs") == 0;', + primary, + ) + self.assertEqual( + primary.count("if (is_preferences) _prefs_load_incomplete = true;"), + 3, + ) + + def test_channel_snapshot_is_failure_atomic_without_large_stack_buffer(self): + load = function_body(self.store, "void DataStore::loadChannels(") + read_at = load.index("for (uint8_t channel_idx = 0;") + apply_at = load.index("host->onChannelLoaded(", read_at) + self.assertLess(read_at, apply_at) + self.assertIn("malloc(sizeof(ChannelDetails) * channel_count)", load) + self.assertNotRegex(load, r"ChannelDetails\s+\w+\s*\[MAX_GROUP_CHANNELS\]") + self.assertIn("channel_count != 0 && loaded == nullptr", load) + self.assertGreaterEqual(load.count("free(loaded);"), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_internal_secondary_fs_repair/test_internal_secondary_fs_repair.cpp b/test/test_internal_secondary_fs_repair/test_internal_secondary_fs_repair.cpp index 07c7d90f0..99d9d78fb 100644 --- a/test/test_internal_secondary_fs_repair/test_internal_secondary_fs_repair.cpp +++ b/test/test_internal_secondary_fs_repair/test_internal_secondary_fs_repair.cpp @@ -5,6 +5,7 @@ using mesh::storage::InternalSecondaryFsRepairResult; using mesh::storage::InternalSecondaryFsBootResult; using mesh::storage::isInternalExtraFsReservedByApplication; +using mesh::storage::isErasedFlashRange; using mesh::storage::isExpectedInternalExtraFsGeometry; using mesh::storage::prepareInternalSecondaryFilesystem; using mesh::storage::repairInternalSecondaryFilesystem; @@ -20,6 +21,36 @@ TEST(InternalSecondaryFsBoot, RequiresExact100KiBGeometry) { EXPECT_FALSE(isInternalExtraFsReservedByApplication(0xED000)); } +TEST(InternalSecondaryFsBoot, ErasedRangeScansEveryWordExactlyOnce) { + const uint32_t start = 0xED000; + const uint32_t size = 7 * 4096; + uint32_t reads = 0; + EXPECT_TRUE(isErasedFlashRange( + start, size, [&](uint32_t address) { + EXPECT_EQ(address, start + reads * sizeof(uint32_t)); + reads++; + return 0xFFFFFFFFUL; + })); + EXPECT_EQ(reads, size / sizeof(uint32_t)); +} + +TEST(InternalSecondaryFsBoot, ErasedRangeRejectsAnyProgrammedWord) { + const uint32_t start = 0xED000; + const uint32_t size = 7 * 4096; + const uint32_t programmed_address = start + size - sizeof(uint32_t); + EXPECT_FALSE(isErasedFlashRange( + start, size, [&](uint32_t address) { + return address == programmed_address ? 0xFFFFFFFEUL : 0xFFFFFFFFUL; + })); +} + +TEST(InternalSecondaryFsBoot, ErasedRangeRejectsIncompleteGeometry) { + auto erased = [](uint32_t) { return 0xFFFFFFFFUL; }; + EXPECT_FALSE(isErasedFlashRange(0xED000, 0, erased)); + EXPECT_FALSE(isErasedFlashRange(0xED001, 7 * 4096, erased)); + EXPECT_FALSE(isErasedFlashRange(0xED000, 7 * 4096 - 1, erased)); +} + TEST(InternalSecondaryFsBoot, HealthyFilesystemMountsWithoutOtherAccess) { int blank_calls = 0; int format_calls = 0; diff --git a/test/test_nrf52_extrafs_contract.py b/test/test_nrf52_extrafs_contract.py index 7f4f66cce..910870e13 100644 --- a/test/test_nrf52_extrafs_contract.py +++ b/test/test_nrf52_extrafs_contract.py @@ -36,6 +36,302 @@ def function_body(source: str, signature: str) -> str: class Nrf52ExtraFsContractTest(unittest.TestCase): + def test_hil_stat_failure_is_consumed_before_source_discovery(self): + store = STORE.read_text(encoding="utf-8") + load_contacts = function_body(store, "void DataStore::loadContacts(") + load_pages = function_body( + store, "bool DataStore::loadContactPages(DataStoreHost* host," + ) + + self.assertEqual(store.count('"/__hil.readfail"'), 1) + self.assertIn("HIL_CONTACT_STAT_FAILURE_FLAG = 0x80", store) + marker_at = load_contacts.index("hil_failure_marker_exists") + remove_at = load_contacts.index( + "contacts_fs->remove(", marker_at + ) + verify_at = load_contacts.index( + "contactPathPresence(", remove_at + ) + presence_at = load_contacts.index("uint32_t contact_page_presence") + source_at = load_contacts.index("chooseContactStoreSource(") + self.assertLess(marker_at, remove_at) + self.assertLess(remove_at, verify_at) + self.assertLess(verify_at, presence_at) + self.assertLess(presence_at, source_at) + + injected_at = load_contacts.index( + "if (page == hil_stat_failure_page)" + ) + incomplete_at = load_contacts.index( + "_contact_load_incomplete = true", injected_at + ) + page_probe_at = load_contacts.index( + "contactPathPresence(contacts_fs, path, page_exists)", injected_at + ) + self.assertLess(injected_at, incomplete_at) + self.assertLess(incomplete_at, page_probe_at) + self.assertLess(page_probe_at, source_at) + + # Low-bit marker values remain handled by the existing post-discovery + # unread-page hook rather than being consumed by the stat hook. + self.assertIn("fail_page == page", load_pages) + self.assertIn("HIL_CONTACT_PAGE_FAILURE_MARKER", load_pages) + + def test_contact_source_presence_uses_raw_tristate_lfs_stat(self): + store = STORE.read_text(encoding="utf-8") + load_contacts = function_body(store, "void DataStore::loadContacts(") + prepare = function_body( + store, "bool DataStore::prepareLegacyContactMigration()" + ) + truncate = function_body( + store, "bool DataStore::truncateLegacyContacts(" + ) + service = function_body( + store, "bool DataStore::serviceContactWrites(DataStoreHost* host," + ) + + stat = function_body( + store, "static mesh::storage::ContactPathState statContactPath(" + ) + self.assertIn("fs->_lockFS();", stat) + self.assertIn("lfs_stat(fs->_getFS(), path, &info)", stat) + self.assertIn("fs->_unlockFS();", stat) + self.assertIn("LFS_ERR_NOENT", stat) + + source_at = load_contacts.index("chooseContactStoreSource(") + page_stat_at = load_contacts.index( + "contactPathPresence(contacts_fs, path, page_exists)" + ) + legacy_stat_at = load_contacts.index( + 'contactPathPresence(contacts_fs, "/contacts3", legacy_exists)' + ) + marker_stat_at = load_contacts.index( + "contactPathPresence(contacts_fs, CONTACT_MIGRATION_MARKER," + ) + incomplete_at = load_contacts.index( + "_contact_load_incomplete = true", page_stat_at + ) + self.assertLess(page_stat_at, incomplete_at) + self.assertLess(incomplete_at, source_at) + self.assertLess(legacy_stat_at, source_at) + self.assertLess(marker_stat_at, source_at) + self.assertNotIn("->exists(", load_contacts) + + for body in (prepare, truncate, service): + self.assertIn("contactPathPresence(", body) + self.assertNotIn("->exists(", body) + + prepare_marker_at = prepare.index( + "contactPathPresence(fs, CONTACT_MIGRATION_MARKER, marker_exists)" + ) + prepare_page_at = prepare.index( + "contactPathPresence(fs, path, path_exists)" + ) + prepare_remove_at = prepare.index("fs->remove(path)") + self.assertLess(prepare_marker_at, prepare_page_at) + self.assertLess(prepare_page_at, prepare_remove_at) + + truncate_legacy_at = truncate.index( + 'contactPathPresence(fs, "/contacts3", legacy_exists)' + ) + truncate_remove_at = truncate.index('fs->remove("/contacts3")') + truncate_marker_at = truncate.index( + "contactPathPresence(fs, CONTACT_MIGRATION_MARKER, marker_exists)" + ) + truncate_complete_at = truncate.index( + "_legacy_contacts_pending_cleanup = false" + ) + self.assertLess(truncate_legacy_at, truncate_remove_at) + self.assertLess(truncate_remove_at, truncate_marker_at) + self.assertLess(truncate_marker_at, truncate_complete_at) + + service_legacy_at = service.index( + 'contactPathPresence(fs, "/contacts3", legacy_exists)' + ) + service_absent_at = service.index("if (!legacy_exists)") + service_marker_at = service.index( + "contactPathPresence(fs, CONTACT_MIGRATION_MARKER," + ) + service_complete_at = service.index( + "_legacy_contacts_pending_cleanup = false" + ) + self.assertLess(service_legacy_at, service_absent_at) + self.assertLess(service_absent_at, service_marker_at) + self.assertLess(service_marker_at, service_complete_at) + + def test_unread_contact_pages_fail_closed_without_rewrite(self): + store = STORE.read_text(encoding="utf-8") + header = STORE_HEADER.read_text(encoding="utf-8") + mesh = MESH.read_text(encoding="utf-8") + + self.assertIn( + "mesh::storage::DirtyPageSet _unread_contact_pages;", header + ) + self.assertIn("bool hasIncompleteContactLoad() const;", header) + + load = function_body( + store, "bool DataStore::loadContactPages(DataStoreHost* host," + ) + quarantine = function_body(load, "auto quarantineUnreadPage =") + self.assertIn("_contact_slots.reserve(", quarantine) + self.assertIn("_unread_contact_pages.mark(page)", quarantine) + self.assertIn("_contact_load_incomplete = true", quarantine) + self.assertGreaterEqual(load.count("quarantineUnreadPage(page)"), 4) + self.assertIn("_dirty_contact_pages.clearAll()", load) + + for signature in ( + "bool DataStore::markContactDirty(const ContactInfo& contact)", + "bool DataStore::releaseContact(const ContactInfo& contact)", + "bool DataStore::restoreContactSlot(const ContactInfo& contact,", + ): + body = function_body(store, signature) + self.assertIn( + "if (hasIncompleteContactLoad()) return false;", body + ) + + service = function_body( + store, "bool DataStore::serviceContactWrites(DataStoreHost* host," + ) + unread_at = service.index("_unread_contact_pages.bits()") + write_at = service.index("writeContactPage(host, page", unread_at) + self.assertLess(unread_at, write_at) + + pending = function_body( + store, "bool DataStore::hasPendingContactWrites() const" + ) + self.assertIn("!hasIncompleteContactLoad()", pending) + + incomplete = function_body( + store, "bool DataStore::hasIncompleteContactLoad() const" + ) + self.assertIn("_contact_load_incomplete", incomplete) + self.assertIn("_unread_contact_pages.empty()", incomplete) + + handler = function_body(mesh, "void MyMesh::handleCmdFrame(") + get_at = handler.index("cmd_frame[0] == CMD_GET_CONTACTS") + next_at = handler.index("cmd_frame[0] == CMD_SET_ADVERT_NAME", get_at) + get_contacts = handler[get_at:next_at] + incomplete_at = get_contacts.index("hasIncompleteContactLoad()") + lock_at = get_contacts.index("_serial->lockReplyRoute()") + self.assertLess(incomplete_at, lock_at) + self.assertIn("ERR_CODE_FILE_IO_ERROR", get_contacts) + + base = (ROOT / "src" / "helpers" / "BaseChatMesh.cpp").read_text( + encoding="utf-8" + ) + advert = function_body(base, "void BaseChatMesh::onAdvertRecv(") + peer_data = function_body(base, "void BaseChatMesh::onPeerDataRecv(") + path = function_body(base, "bool BaseChatMesh::onContactPathRecv(") + self.assertIn("if (!canMutateContacts())", advert) + self.assertGreaterEqual(peer_data.count("canMutateContacts()"), 2) + self.assertIn("if (canMutateContacts())", path) + self.assertIn( + "bool MyMesh::canMutateContacts() const", mesh + ) + + terminal_path = function_body(mesh, "void MyMesh::handleTerminalPath(") + self.assertIn("const ContactInfo previous = recipient", terminal_path) + self.assertIn("if (!scheduleContactWrite(recipient))", terminal_path) + self.assertIn("recipient = previous", terminal_path) + + terminal = function_body(mesh, "void MyMesh::handleTerminalCommand(") + reset_at = terminal.index('strcmp(command, "reset path")') + next_at = terminal.index('strcmp(command, "card")', reset_at) + reset_path = terminal[reset_at:next_at] + self.assertIn("previous_out_path_len", reset_path) + self.assertIn("if (scheduleContactWrite(*recipient))", reset_path) + self.assertIn( + "recipient->out_path_len = previous_out_path_len", reset_path + ) + + def test_legacy_and_first_pass_page_load_fail_closed(self): + store = STORE.read_text(encoding="utf-8") + load_contacts = function_body(store, "void DataStore::loadContacts(") + load_pages = function_body( + store, "bool DataStore::loadContactPages(DataStoreHost* host," + ) + + self.assertIn("uint32_t contact_page_presence = 0", load_contacts) + self.assertIn("contact_page_presence |= 1UL << page", load_contacts) + self.assertIn( + "loadContactPages(host, 0, contact_page_presence)", load_contacts + ) + self.assertIn("expected_page_mask", load_pages) + self.assertIn( + "if ((expected_page_mask & (1UL << page)) == 0) continue;", + load_pages, + ) + self.assertNotIn("if (!fs->exists(path)) continue;", load_pages) + open_failure = load_pages.index("if (!file)") + quarantine = load_pages.index("quarantineUnreadPage(page)", open_failure) + self.assertLess(open_failure, quarantine) + + legacy_open = load_contacts.index('openRead(_getContactsChannelsFS(), "/contacts3")') + open_failure = load_contacts.index("if (!file)", legacy_open) + open_incomplete = load_contacts.index( + "_contact_load_incomplete = true", open_failure + ) + size_check = load_contacts.index( + "isValidLegacyContactFileSize(legacy_size)", open_incomplete + ) + read_failure = load_contacts.index("legacy_read_failed = true", size_check) + host_failure = load_contacts.index("legacy_host_refused = true", read_failure) + incomplete_return = load_contacts.index( + "_contact_load_incomplete = true", host_failure + ) + migration_prepare = load_contacts.index( + "prepareLegacyContactMigration()", incomplete_return + ) + self.assertLess(open_failure, open_incomplete) + self.assertLess(size_check, read_failure) + self.assertLess(read_failure, host_failure) + self.assertLess(incomplete_return, migration_prepare) + self.assertIn("record_index != _legacy_contact_count", load_contacts) + self.assertIn("contact_page_presence = 0", load_contacts) + + reset = function_body(store, "void DataStore::resetContactPageState(") + self.assertIn("if (clear_incomplete)", reset) + reset_at = load_contacts.index("resetContactPageState()") + latched_at = load_contacts.index( + "if (_contact_load_incomplete)", reset_at + ) + presence_at = load_contacts.index( + "uint32_t contact_page_presence", latched_at + ) + self.assertLess(reset_at, latched_at) + self.assertLess(latched_at, presence_at) + + service = function_body( + store, "bool DataStore::serviceContactWrites(DataStoreHost* host," + ) + self.assertLess( + service.index("if (hasIncompleteContactLoad())"), + service.index("_legacy_contacts_pending_cleanup"), + ) + + def test_contact_import_is_rejected_while_load_is_incomplete(self): + mesh = MESH.read_text(encoding="utf-8") + handler = function_body(mesh, "void MyMesh::handleCmdFrame(") + import_at = handler.index("cmd_frame[0] == CMD_IMPORT_CONTACT") + next_at = handler.index("cmd_frame[0] == CMD_SYNC_NEXT_MESSAGE", import_at) + binary_import = handler[import_at:next_at] + gate = binary_import.index("hasIncompleteContactLoad()") + error = binary_import.index("ERR_CODE_FILE_IO_ERROR", gate) + enqueue = binary_import.index("importContact(", error) + ok = binary_import.index("writeOKFrame()", enqueue) + self.assertLess(gate, error) + self.assertLess(error, enqueue) + self.assertLess(enqueue, ok) + + terminal_import = function_body( + mesh, "void MyMesh::importTerminalCard(" + ) + gate = terminal_import.index("hasIncompleteContactLoad()") + retry = terminal_import.index("reboot and retry", gate) + enqueue = terminal_import.index("importContact(", retry) + self.assertLess(gate, retry) + self.assertLess(retry, enqueue) + def test_t1000_usb_and_ble_reserve_exact_100k_region(self): profile = PROFILE.read_text(encoding="utf-8") for environment in ( @@ -82,7 +378,9 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): ) self.assertNotIn("ExtraFS.begin()", setup) self.assertIn("PreservedNonBlank", setup) - self.assertIn("store.disableSecondaryFS();", setup) + self.assertIn( + "store.disableSecondaryFS(secondary_authority_unknown);", setup + ) self.assertLess( setup.index("prepareInternalSecondaryFilesystem("), setup.index("store.begin()"), @@ -98,7 +396,7 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): re.DOTALL, ) self.assertIsNotNone(invalid) - self.assertIn("_fsExtra = nullptr;", invalid.group(1)) + self.assertIn("disableSecondaryFS(true);", invalid.group(1)) self.assertNotIn("format", invalid.group(1)) self.assertNotIn("reinitializeInternalExtraFS", invalid.group(1)) @@ -186,20 +484,177 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): mesh = MESH.read_text(encoding="utf-8") repair = function_body(mesh, "void MyMesh::repairInternalExtraFS(") + quarantine_at = repair.index("_store->hasIncompleteContactLoad()") migrate_at = repair.index("_store->repairInternalExtraFS()") - return_at = repair.index("return;", migrate_at) + reload_gate = repair.index("if (authoritative_reload_required)", migrate_at) + reboot_at = repair.index("_scheduled_reboot_at = futureMillis(1000)", reload_gate) + return_at = repair.index("return;", reboot_at) contacts_at = repair.index("_store->saveContacts(this, save_filter)") + self.assertLess(quarantine_at, migrate_at) self.assertLess(migrate_at, return_at) + self.assertLess(reload_gate, reboot_at) + self.assertLess(reboot_at, return_at) self.assertLess(return_at, contacts_at) self.assertNotIn("already_ready", repair) + def test_secondary_migration_presence_errors_fail_before_authority_changes(self): + store = STORE.read_text(encoding="utf-8") + migration = function_body(store, "bool DataStore::migrateToSecondaryFS()") + + copy_helper = function_body(migration, "auto copy =") + self.assertIn("bool exact_snapshot", migration) + source_stat_at = copy_helper.index( + "contactPathPresence(source_fs, path, source_exists)" + ) + destination_stat_at = copy_helper.index( + "contactPathPresence(dest_fs, path, destination_exists)" + ) + destination_open_at = copy_helper.index( + "mesh::AtomicFileWriter destination(dest_fs, path)" + ) + self.assertLess(source_stat_at, destination_stat_at) + self.assertLess(destination_stat_at, destination_open_at) + self.assertNotIn("->exists(", copy_helper[: copy_helper.index("#else")]) + + source_absent_at = copy_helper.index("if (!source_exists)") + orphan_remove_at = copy_helper.index("dest_fs->remove(path)", source_absent_at) + verified_absent_at = copy_helper.index( + "contactPathPresence(dest_fs, path, destination_remains)", + orphan_remove_at, + ) + self.assertLess(source_absent_at, orphan_remove_at) + self.assertLess(orphan_remove_at, verified_absent_at) + conflict_at = copy_helper.index("if (!exact_snapshot)", verified_absent_at) + self.assertLess(conflict_at, destination_open_at) + + primary_helper = function_body( + migration, "auto migratePrimarySources =" + ) + self.assertIn( + "contactPathPresence(_fsExtra, path, secondary_exists)", + primary_helper, + ) + self.assertIn( + "contactPathPresence(_fs, path, primary_exists)", primary_helper + ) + self.assertNotIn( + "->exists(", primary_helper[: primary_helper.index("#else")] + ) + + read_journal = function_body(migration, "auto readJournal =") + journal_stat_at = read_journal.index( + "contactPathPresence(_fsExtra, SECONDARY_MIGRATION_JOURNAL," + ) + journal_open_at = read_journal.index( + "openRead(_fsExtra, SECONDARY_MIGRATION_JOURNAL)" + ) + self.assertLess(journal_stat_at, journal_open_at) + self.assertIn("return MigrationJournalState::IoError;", read_journal) + self.assertNotIn("->exists(", read_journal) + + source_scan = function_body(migration, "auto hasSecondarySources =") + self.assertGreaterEqual(source_scan.count("contactPathPresence("), 3) + self.assertIn("present = false;", source_scan) + self.assertIn("return false;", source_scan) + self.assertNotIn("->exists(", source_scan) + + copy_sources = function_body(migration, "auto copySecondarySources =") + self.assertEqual(copy_sources.count(", true)"), 3) + + pending_at = migration.index( + "writeJournal(SECONDARY_MIGRATION_PENDING)" + ) + scan_at = migration.index( + "!hasSecondarySources(secondary_sources_present)" + ) + self.assertLess(scan_at, pending_at) + scan_failure_end = migration.index( + "if (journal_state != MigrationJournalState::Committed", scan_at + ) + scan_failure_branch = migration[scan_at:scan_failure_end] + self.assertIn("_contact_load_incomplete = true;", scan_failure_branch) + self.assertIn("return false;", scan_failure_branch) + self.assertNotIn("_fsExtra = nullptr;", scan_failure_branch) + + journal_io_at = migration.index( + "journal_state == MigrationJournalState::IoError" + ) + journal_io_end = migration.index( + "if (journal_state == MigrationJournalState::Invalid)", + journal_io_at, + ) + journal_io_branch = migration[journal_io_at:journal_io_end] + self.assertIn("_contact_load_incomplete = true;", journal_io_branch) + self.assertIn("return false;", journal_io_branch) + self.assertNotIn("_fsExtra = nullptr;", journal_io_branch) + + primary_failure_at = migration.index("if (!migratePrimarySources())") + self.assertLess( + migration.index("MigrationJournalState journal_state = readJournal()"), + primary_failure_at, + ) + self.assertLess(primary_failure_at, journal_io_at) + primary_failure_branch = migration[primary_failure_at:journal_io_at] + self.assertIn("_contact_load_incomplete = true;", primary_failure_branch) + self.assertIn("return false;", primary_failure_branch) + self.assertNotIn("_fsExtra = nullptr;", primary_failure_branch) + + invalid_at = migration.index( + "journal_state == MigrationJournalState::Invalid" + ) + invalid_end = migration.index( + "bool secondary_sources_present", invalid_at + ) + invalid_branch = migration[invalid_at:invalid_end] + self.assertIn("_contact_load_incomplete = true;", invalid_branch) + self.assertIn("return false;", invalid_branch) + self.assertNotIn("_fsExtra = nullptr;", invalid_branch) + + retire_helper = function_body( + migration, "auto retireSecondarySources =" + ) + last_stat_at = retire_helper.rindex("contactPathPresence(") + first_remove_at = retire_helper.index("_fs->remove(path)") + self.assertLess(last_stat_at, first_remove_at) + self.assertIn("uint32_t page_presence = 0;", retire_helper) + self.assertIn("uint16_t bucket_presence = 0;", retire_helper) + self.assertNotIn("->exists(", retire_helper) + + committed_at = migration.index( + "writeJournal(SECONDARY_MIGRATION_COMMITTED)" + ) + retire_at = migration.index("retireSecondarySources()", committed_at) + journal_remove_at = migration.index( + "_fsExtra->remove(SECONDARY_MIGRATION_JOURNAL)", retire_at + ) + self.assertLess(committed_at, retire_at) + self.assertLess(retire_at, journal_remove_at) + self.assertNotIn( + "_fsExtra->exists(SECONDARY_MIGRATION_JOURNAL)", migration + ) + + load_channels = function_body( + store, "void DataStore::loadChannels(DataStoreHost* host)" + ) + save_channels = function_body( + store, "bool DataStore::saveChannels(DataStoreHost* host)" + ) + self.assertLess( + load_channels.index("if (_contact_load_incomplete)"), + load_channels.index('openRead(_getContactsChannelsFS(), "/channels2")'), + ) + self.assertLess( + save_channels.index("if (_contact_load_incomplete) return false;"), + save_channels.index("mesh::AtomicFileWriter file("), + ) + def test_legacy_primary_files_move_before_contact_journal_can_fall_back(self): store = STORE.read_text(encoding="utf-8") migration = function_body(store, "bool DataStore::migrateToSecondaryFS()") # Older ExtraFS test builds could leave identity and preferences only - # on the secondary filesystem. Recover those first: every journal - # failure below disables/falls back from that secondary filesystem. + # on the secondary filesystem. Read the journal without changing + # authority, then recover these files before acting on that result. self.assertIn( 'static const char* to_primary[] = {"/_main.id", "/new_prefs"};', migration, @@ -208,7 +663,12 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): journal_flow_at = migration.index( "MigrationJournalState journal_state = readJournal()" ) - self.assertLess(primary_move_at, journal_flow_at) + journal_result_at = migration.index( + "journal_state == MigrationJournalState::IoError", + journal_flow_at, + ) + self.assertLess(journal_flow_at, primary_move_at) + self.assertLess(primary_move_at, journal_result_at) # A primary-present conflict is nonfatal: primary is canonical and the # differing secondary copy is preserved for recovery. Only failure to @@ -226,7 +686,7 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): "if (!filesEqual(_fsExtra, _fs, path))" ) missing_copy_at = primary_helper.index( - "else if (!copy(_fsExtra, _fs, path))" + "bool recovered_primary = copy(_fsExtra, _fs, path, false)" ) remove_at = primary_helper.index("_fsExtra->remove(path)") self.assertLess(conflict_at, missing_copy_at) @@ -246,6 +706,197 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): "preserving conflicting legacy secondary", primary_helper ) + def test_identity_creation_requires_proven_absence(self): + store = STORE.read_text(encoding="utf-8") + header = STORE_HEADER.read_text(encoding="utf-8") + mesh = MESH.read_text(encoding="utf-8") + + self.assertIn("bool _identity_creation_blocked = false;", header) + self.assertIn("bool _primary_storage_unavailable = false;", header) + self.assertIn("bool _secondary_authority_unknown = false;", header) + self.assertIn("bool canCreateMainIdentity() const;", header) + self.assertIn( + "void disableSecondaryFS(bool authority_unknown = true);", header + ) + self.assertIn("void markPrimaryFSUnavailable();", header) + + load = function_body( + store, + "bool DataStore::loadMainIdentity(mesh::LocalIdentity &identity)", + ) + raw_stat = load.index( + 'contactPathPresence(_fs, "/_main.id", identity_exists)' + ) + proven_missing = load.index("if (!identity_exists) return false;", raw_stat) + decode = load.index('identity_store.load("_main", identity)', proven_missing) + self.assertLess(raw_stat, proven_missing) + self.assertLess(proven_missing, decode) + self.assertIn( + "_identity_creation_blocked = true;", + load[raw_stat:proven_missing], + ) + self.assertIn( + "_identity_creation_blocked = true;", + load[decode:], + ) + self.assertIn("_identity_creation_blocked = false;", load[decode:]) + + save = function_body( + store, + "bool DataStore::saveMainIdentity(const mesh::LocalIdentity &identity)", + ) + self.assertLess( + save.index("if (_identity_creation_blocked) return false;"), + save.index('identity_store.save("_main", identity)'), + ) + + migration = function_body( + store, "bool DataStore::migrateToSecondaryFS()" + ) + primary_helper = function_body( + migration, "auto migratePrimarySources =" + ) + self.assertIn( + 'const bool is_identity = strcmp(path, "/_main.id") == 0;', + primary_helper, + ) + self.assertGreaterEqual( + primary_helper.count( + "if (is_identity) _identity_creation_blocked = true;" + ), + 3, + ) + recovered = primary_helper.index( + "bool recovered_primary = copy(_fsExtra, _fs, path, false)" + ) + verify_destination = primary_helper.index( + "contactPathPresence(_fs, path, recovered_primary_exists)", + recovered, + ) + reject_missing = primary_helper.index( + "if (!recovered_primary)", verify_destination + ) + self.assertLess(recovered, verify_destination) + self.assertLess(verify_destination, reject_missing) + + store_begin = function_body(store, "void DataStore::begin()") + reset_at = store_begin.index("resetContactPageState(true)") + unmounted_at = store_begin.index("if (_primary_storage_unavailable)") + primary_validate = store_begin.index("validateLfsFilesystem(_fs)") + primary_failure = store_begin.index("if (!primary_ready)", primary_validate) + secondary_validate = store_begin.index( + "validateLfsFilesystem(_fsExtra)", primary_failure + ) + primary_failure_branch = store_begin[primary_failure:secondary_validate] + self.assertLess(reset_at, primary_validate) + self.assertLess(unmounted_at, primary_validate) + unmounted_branch = store_begin[unmounted_at:primary_validate] + self.assertIn("_identity_creation_blocked = true;", unmounted_branch) + self.assertIn("_contact_load_incomplete = true;", unmounted_branch) + self.assertIn("return;", unmounted_branch) + self.assertIn("_primary_storage_unavailable = true;", primary_failure_branch) + self.assertIn("_identity_creation_blocked = true;", primary_failure_branch) + self.assertIn("_contact_load_incomplete = true;", primary_failure_branch) + self.assertIn("return;", primary_failure_branch) + self.assertNotIn("recoverPrimaryFilesystem", store_begin) + self.assertNotIn("->format(", store_begin) + secondary_failure = store_begin.index( + "if (_fsExtra != nullptr && !validateLfsFilesystem(_fsExtra))" + ) + self.assertIn( + "disableSecondaryFS(true);", + store_begin[secondary_failure:], + ) + + disable = function_body( + store, "void DataStore::disableSecondaryFS(bool authority_unknown)" + ) + self.assertIn("_secondary_authority_unknown = true;", disable) + self.assertIn("_contact_load_incomplete = true;", disable) + self.assertIn("_identity_creation_blocked = true;", disable) + + unavailable = function_body( + store, "void DataStore::markPrimaryFSUnavailable()" + ) + self.assertIn("_primary_storage_unavailable = true;", unavailable) + self.assertIn("_contact_load_incomplete = true;", unavailable) + self.assertIn("_identity_creation_blocked = true;", unavailable) + + setup = function_body(MAIN.read_text(encoding="utf-8"), "void setup()") + main = MAIN.read_text(encoding="utf-8") + self.assertIn( + "static const uint32_t INTERNAL_PRIMARY_FS_START = 0x000ED000UL;", + main, + ) + self.assertIn( + "static const uint32_t INTERNAL_PRIMARY_FS_START = 0x0006D000UL;", + main, + ) + self.assertIn("7UL * FLASH_NRF52_PAGE_SIZE", main) + primary_policy = setup.index( + "InternalSecondaryFsBootResult primary_fs_boot" + ) + primary_mount = setup.index( + "InternalFS.Adafruit_LittleFS::begin()", primary_policy + ) + blank_scan = setup.index("isErasedFlashRange(", primary_mount) + primary_format = setup.index("InternalFS.format()", blank_scan) + quarantine = setup.index("store.markPrimaryFSUnavailable()", primary_format) + store_begin = setup.index("store.begin()", quarantine) + self.assertLess(primary_policy, primary_mount) + self.assertLess(primary_mount, blank_scan) + self.assertLess(blank_scan, primary_format) + self.assertLess(primary_format, quarantine) + self.assertLess(quarantine, store_begin) + primary_policy_body = setup[primary_policy:quarantine] + self.assertIn("INTERNAL_PRIMARY_FS_START", primary_policy_body) + self.assertIn("INTERNAL_PRIMARY_FS_SIZE", primary_policy_body) + self.assertIn( + "InternalSecondaryFsBootResult::PreservedNonBlank", + primary_policy_body, + ) + self.assertIn( + "InternalSecondaryFsBootResult::InitializationFailed", + primary_policy_body, + ) + # Never call the core override in the nRF path: it auto-erases a + # nonblank filesystem after any mount failure. The sole occurrence is + # the separately scoped STM32 fallback. + self.assertEqual(setup.count("InternalFS.begin();"), 1) + self.assertGreater(setup.index("#else", primary_policy), primary_format) + self.assertIn("store.disableSecondaryFS(true);", setup) + authority = setup.index( + "const bool secondary_authority_unknown = !extra_fs_geometry_valid" + ) + disable_call = setup.index( + "store.disableSecondaryFS(secondary_authority_unknown)", authority + ) + authority_expression = setup[authority:disable_call] + self.assertIn( + "InternalSecondaryFsBootResult::PreservedNonBlank", + authority_expression, + ) + self.assertNotIn( + "InternalSecondaryFsBootResult::InitializationFailed", + authority_expression, + ) + + begin = function_body(mesh, "void MyMesh::begin(") + load_at = begin.index("const bool identity_loaded =") + creation_gate = begin.index( + "is_new_install && !_store->canCreateMainIdentity()", load_at + ) + generate_at = begin.index("mesh::generateUsableLocalIdentity", creation_gate) + save_at = begin.index( + "identity_ready = _store->saveMainIdentity(self_id)", generate_at + ) + self.assertLess(load_at, creation_gate) + self.assertLess(creation_gate, generate_at) + self.assertLess(generate_at, save_at) + blocked_branch = begin[creation_gate:generate_at] + self.assertIn("board.reboot();", blocked_branch) + self.assertIn("return;", blocked_branch) + def test_local_rescue_repair_rebuilds_live_contacts_and_channels(self): mesh = MESH.read_text(encoding="utf-8") repair = function_body(mesh, "void MyMesh::repairInternalExtraFS(") @@ -324,6 +975,79 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): self.assertLess(rebuild.index(call), success_at) self.assertIn("Error:", rebuild[success_at:]) + def test_channel_update_rolls_back_when_atomic_save_fails(self): + mesh = MESH.read_text(encoding="utf-8") + handler = function_body(mesh, "void MyMesh::handleCmdFrame(") + channel_at = handler.index( + "cmd_frame[0] == CMD_SET_CHANNEL && len >= 2 + 32 + 16" + ) + next_command_at = handler.index( + "cmd_frame[0] == CMD_SIGN_START", channel_at + ) + channel_update = handler[channel_at:next_command_at] + + snapshot_at = channel_update.index( + "getChannel(channel_idx, previous)" + ) + mutate_at = channel_update.index( + "setChannel(channel_idx, channel)" + ) + save_at = channel_update.index("saveChannels()") + rollback_at = channel_update.index( + "setChannel(channel_idx, previous)", save_at + ) + error_at = channel_update.index( + "writeErrFrame(ERR_CODE_FILE_IO_ERROR)", rollback_at + ) + ok_at = channel_update.index("writeOKFrame()") + + self.assertLess(snapshot_at, mutate_at) + self.assertLess(mutate_at, save_at) + self.assertLess(save_at, ok_at) + self.assertLess(save_at, rollback_at) + self.assertLess(rollback_at, error_at) + + def test_absent_advert_delete_avoids_bucket_rewrite_without_legacy_cache(self): + store = STORE.read_text(encoding="utf-8") + body = function_body(store, "bool DataStore::deleteBlobByKey(") + fast_path = body.index( + "if (!legacy_exists && bucket_valid" + ) + save = body.index("saveBlobBucket(") + self.assertLess(fast_path, save) + self.assertIn( + "&& (!found || records[selected].len == 0)", + body[fast_path:save], + ) + self.assertIn("free(records);", body[fast_path:save]) + self.assertIn("return true;", body[fast_path:save]) + + def test_nrf_advert_tombstones_never_fall_back_to_legacy_cache(self): + store = STORE.read_text(encoding="utf-8") + read = function_body(store, "uint8_t DataStore::getBlobByKey(") + bucket_at = read.index("findBlobInBucket(") + nrf_return_at = read.index("return 0;", bucket_at) + legacy_at = read.index( + 'File file = openRead(_getContactsChannelsFS(), "/adv_blobs")' + ) + self.assertLess(nrf_return_at, legacy_at) + between = read[nrf_return_at:legacy_at] + self.assertIn("#endif", between) + self.assertIn("#if !defined(NRF52_PLATFORM)", between) + + put = function_body(store, "bool DataStore::putBlobByKey(") + cleanup_at = put.index("clearLegacyBlobRecord(") + self.assertNotIn( + "return false;", put[cleanup_at:put.index("#else", cleanup_at)] + ) + + delete = function_body(store, "bool DataStore::deleteBlobByKey(") + cleanup_at = delete.index("clearLegacyBlobRecord(") + self.assertNotIn( + "return false;", + delete[cleanup_at:delete.index("#else", cleanup_at)], + ) + def test_quarantined_extrafs_rescue_file_commands_are_null_safe(self): mesh = MESH.read_text(encoding="utf-8") rescue = function_body(mesh, "void MyMesh::checkCLIRescueCmd()") diff --git a/test/test_persistent_store_format/test_persistent_store_format.cpp b/test/test_persistent_store_format/test_persistent_store_format.cpp index 5389f7800..bef87b2ad 100644 --- a/test/test_persistent_store_format/test_persistent_store_format.cpp +++ b/test/test_persistent_store_format/test_persistent_store_format.cpp @@ -21,7 +21,8 @@ TEST(PersistentStoreFormat, ContactHeaderRoundTripsAndRejectsDamage) { raw[0] ^= 1; EXPECT_FALSE(decodeContactPageHeader(raw, 2, decoded)); writeLE32(&raw[8], 1UL << CONTACTS_PER_PAGE); - EXPECT_FALSE(decodeContactPageHeader(raw, 3, decoded)); + EXPECT_TRUE(decodeContactPageHeader(raw, 3, decoded)); + EXPECT_NE(decoded.occupied & ~contactPageValidSlotMask(), 0u); } TEST(PersistentStoreFormat, CRCDetectsPayloadChanges) { @@ -35,6 +36,18 @@ TEST(PersistentStoreFormat, CRCDetectsPayloadChanges) { EXPECT_EQ(split, expected); } +TEST(PersistentStoreFormat, OccupancyCanBeDerivedFromCrcProtectedRecords) { + uint8_t empty[CONTACT_RECORD_SIZE] = {}; + EXPECT_FALSE(contactRecordHasData(empty)); + + for (uint16_t offset : {uint16_t(0), uint16_t(31), + uint16_t(CONTACT_RECORD_SIZE - 1)}) { + uint8_t populated[CONTACT_RECORD_SIZE] = {}; + populated[offset] = 1; + EXPECT_TRUE(contactRecordHasData(populated)); + } +} + TEST(PersistentStoreFormat, DirtyPagesStayPendingUntilExplicitlyCleared) { DirtyPageSet pages; EXPECT_TRUE(pages.empty()); @@ -55,6 +68,18 @@ TEST(PersistentStoreFormat, LegacyCountIgnoresPartialTailAndCapsAtCapacity) { EXPECT_EQ(legacyContactCountForSize(999 * CONTACT_RECORD_SIZE), 350); } +TEST(PersistentStoreFormat, LegacyFileSizeValidationRejectsUnsafeSources) { + const size_t capacity = CONTACT_PAGE_COUNT * CONTACTS_PER_PAGE; + EXPECT_TRUE(isValidLegacyContactFileSize(0)); + EXPECT_TRUE(isValidLegacyContactFileSize(CONTACT_RECORD_SIZE)); + EXPECT_TRUE(isValidLegacyContactFileSize(capacity * CONTACT_RECORD_SIZE)); + EXPECT_FALSE(isValidLegacyContactFileSize(CONTACT_RECORD_SIZE - 1)); + EXPECT_FALSE(isValidLegacyContactFileSize( + 63 * CONTACT_RECORD_SIZE + 17)); + EXPECT_FALSE(isValidLegacyContactFileSize( + (capacity + 1) * CONTACT_RECORD_SIZE)); +} + TEST(PersistentStoreFormat, LegacyMigrationMovesTailOnePageAtATime) { EXPECT_EQ(legacyMigrationPage(63), 2); EXPECT_EQ(legacyCountAfterMigratingPage(2), 50); @@ -107,6 +132,40 @@ TEST(PersistentStoreFormat, SlotAllocationIsStableAndReusable) { EXPECT_FALSE(slots.release(CONTACT_PAGE_COUNT * CONTACTS_PER_PAGE)); } +TEST(PersistentStoreFormat, ReleasedSlotCanBeRestoredPastAnEarlierHole) { + ContactSlotMap slots; + ASSERT_TRUE(slots.reserve(0)); + ASSERT_TRUE(slots.reserve(CONTACTS_PER_PAGE)); + ASSERT_TRUE(slots.release(CONTACTS_PER_PAGE)); + + // Slot 1 is the allocator's earliest hole, but transaction rollback must be + // able to reclaim the exact page-1 slot which was just released. + EXPECT_TRUE(slots.reserve(CONTACTS_PER_PAGE)); + EXPECT_FALSE(slots.isUsed(1)); + EXPECT_TRUE(slots.isUsed(CONTACTS_PER_PAGE)); + EXPECT_EQ(slots.pageMask(0), 1u); + EXPECT_EQ(slots.pageMask(1), 1u); +} + +TEST(PersistentStoreFormat, FailedSlotAllocationAndReleaseAreNonMutating) { + ContactSlotMap slots; + for (uint16_t slot = 0; + slot < CONTACT_PAGE_COUNT * CONTACTS_PER_PAGE; slot++) { + ASSERT_EQ(slots.allocate(), slot); + } + + uint32_t full_masks[CONTACT_PAGE_COUNT]; + for (uint8_t page = 0; page < CONTACT_PAGE_COUNT; page++) { + full_masks[page] = slots.pageMask(page); + } + + EXPECT_EQ(slots.allocate(), CONTACT_SLOT_NONE); + EXPECT_FALSE(slots.release(CONTACT_SLOT_NONE)); + for (uint8_t page = 0; page < CONTACT_PAGE_COUNT; page++) { + EXPECT_EQ(slots.pageMask(page), full_masks[page]); + } +} + TEST(PersistentStoreFormat, LegacyMarkerSelectsResumableMigrationPath) { EXPECT_EQ(chooseContactStoreSource(true, true), ContactStoreSource::LEGACY); EXPECT_EQ(chooseContactStoreSource(true, false), ContactStoreSource::LEGACY); @@ -118,6 +177,54 @@ TEST(PersistentStoreFormat, LegacyMarkerSelectsResumableMigrationPath) { EXPECT_TRUE(trustMigratedContactPages(false, false)); } +TEST(PersistentStoreFormat, RawPathStatDistinguishesAbsenceFromIoFailure) { + constexpr int no_entry = -2; + EXPECT_EQ(classifyContactPathStat(0, no_entry), ContactPathState::PRESENT); + EXPECT_EQ(classifyContactPathStat(no_entry, no_entry), + ContactPathState::ABSENT); + EXPECT_EQ(classifyContactPathStat(-5, no_entry), + ContactPathState::IO_ERROR); + EXPECT_EQ(classifyContactPathStat(1, no_entry), + ContactPathState::IO_ERROR); +} + +TEST(PersistentStoreFormat, MigrationPresenceScanFailsClosedOnAnyIoError) { + constexpr int no_entry = -2; + const int empty_scan[] = {no_entry, no_entry, no_entry}; + bool empty_has_source = false; + for (int result : empty_scan) { + const ContactPathState state = classifyContactPathStat(result, no_entry); + ASSERT_NE(state, ContactPathState::IO_ERROR); + empty_has_source = empty_has_source || state == ContactPathState::PRESENT; + } + EXPECT_FALSE(empty_has_source); + + const int populated_scan[] = {no_entry, 0, no_entry}; + bool populated_has_source = false; + for (int result : populated_scan) { + const ContactPathState state = classifyContactPathStat(result, no_entry); + ASSERT_NE(state, ContactPathState::IO_ERROR); + populated_has_source = + populated_has_source || state == ContactPathState::PRESENT; + } + EXPECT_TRUE(populated_has_source); + + const int failed_scan[] = {no_entry, -5, 0}; + bool scan_ok = true; + bool failed_has_source = false; + for (int result : failed_scan) { + const ContactPathState state = classifyContactPathStat(result, no_entry); + if (state == ContactPathState::IO_ERROR) { + scan_ok = false; + break; + } + failed_has_source = + failed_has_source || state == ContactPathState::PRESENT; + } + EXPECT_FALSE(scan_ok); + EXPECT_FALSE(failed_has_source); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/test_serial_mode_switch/test_serial_mode_switch.cpp b/test/test_serial_mode_switch/test_serial_mode_switch.cpp index fd379b07f..b3548f5ca 100644 --- a/test/test_serial_mode_switch/test_serial_mode_switch.cpp +++ b/test/test_serial_mode_switch/test_serial_mode_switch.cpp @@ -51,6 +51,11 @@ public: static const char START_TOKEN[] = "+++MESHCORE-TERM-START"; static const char SEEDER_TOKEN[] = "ota folder on"; +static bool serial_client_connected = false; + +static bool isSerialClientConnected() { + return serial_client_connected; +} TEST(UsbHwcdcSession, IgnoresWholeExpectedSelfResetBurstUntilActivity) { mesh::UsbSelfResetBurstGuard guard; @@ -826,6 +831,37 @@ TEST(SerialFlowControl, KeepsAFrameQueuedUntilUsbHasSpace) { EXPECT_FALSE(interface.hasPendingIO()); } +TEST(SerialFlowControl, DisconnectedRequiredFrameReportsFailure) { + BufferStream stream; + ArduinoSerialInterface interface; + interface.begin(stream); + interface.enableFlowControl(true); + interface.setConnectedCheck(isSerialClientConnected); + interface.enable(); + serial_client_connected = false; + + const uint8_t response[] = {0x03, 0x42}; + EXPECT_EQ(interface.writeFrame(response, sizeof(response)), 0u); + EXPECT_TRUE(stream.output.empty()); + EXPECT_FALSE(interface.hasPendingIO()); +} + +TEST(SerialFlowControl, DisconnectedBestEffortPushMayBeDropped) { + BufferStream stream; + ArduinoSerialInterface interface; + interface.begin(stream); + interface.enableFlowControl(true); + interface.setConnectedCheck(isSerialClientConnected); + interface.enable(); + serial_client_connected = false; + + const uint8_t advert_push[] = {0x80, 0x42}; + EXPECT_EQ(interface.writeFrame(advert_push, sizeof(advert_push)), + sizeof(advert_push)); + EXPECT_TRUE(stream.output.empty()); + EXPECT_FALSE(interface.hasPendingIO()); +} + TEST(SerialFlowControl, FinishesAnUnexpectedShortWriteBeforeNextFrame) { BufferStream stream; stream.write_capacity = MAX_FRAME_SIZE + 3; diff --git a/test/test_t1000e_contact_protocol_stress.py b/test/test_t1000e_contact_protocol_stress.py new file mode 100644 index 000000000..c752e8a15 --- /dev/null +++ b/test/test_t1000e_contact_protocol_stress.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""Offline behavioral tests for the focused T1000-E protocol HIL tool.""" + +import argparse +import asyncio +import importlib.util +from pathlib import Path +import struct +import sys +from types import SimpleNamespace +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +HIL_DIR = ROOT / "tools" / "hil" +if str(HIL_DIR) not in sys.path: + sys.path.insert(0, str(HIL_DIR)) +TOOL_PATH = HIL_DIR / "t1000e_contact_protocol_stress.py" +SPEC = importlib.util.spec_from_file_location( + "t1000e_contact_protocol_stress", TOOL_PATH +) +assert SPEC is not None and SPEC.loader is not None +HIL = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = HIL +SPEC.loader.exec_module(HIL) + +TEST_NODE_KEY_PREFIX = "01234567" + + +def valid_config(**changes): + values = dict( + port="COM23", + allow_destructive=True, + confirm_usb_serial=HIL.EXPECTED_USB_SERIAL, + expected_node_key_prefix=TEST_NODE_KEY_PREFIX, + expected_contact_keyset_sha256=HIL.EXPECTED_KEYSET_SHA256, + mutation_index=349, + stream_trigger_contacts=12, + settle_seconds=7.0, + verbose=False, + ) + values.update(changes) + return HIL.Config(**values) + + +class SafetyGateTests(unittest.TestCase): + def test_all_destructive_confirmations_are_mandatory(self): + bad = ( + valid_config(allow_destructive=False), + valid_config(confirm_usb_serial="WRONG"), + valid_config(expected_node_key_prefix="deadbee"), + valid_config(expected_node_key_prefix="not-hex!"), + valid_config(expected_contact_keyset_sha256="00" * 32), + ) + for config in bad: + with self.subTest(config=config): + with self.assertRaises(HIL.ProtocolHilFailure): + HIL.require_consent(config) + + HIL.require_consent(valid_config()) + + def test_ranges_fail_closed(self): + for config in ( + valid_config(mutation_index=-1), + valid_config(mutation_index=350), + valid_config(stream_trigger_contacts=0), + valid_config(stream_trigger_contacts=101), + valid_config(settle_seconds=0), + ): + with self.subTest(config=config): + with self.assertRaises(HIL.ProtocolHilFailure): + HIL.require_consent(config) + + def test_port_is_required_outside_self_test(self): + parser = HIL.build_argument_parser() + args = parser.parse_args([]) + with self.assertRaisesRegex(HIL.ProtocolHilFailure, "--port"): + HIL.config_from_args(args) + + +class ContactFrameTests(unittest.TestCase): + def setUp(self): + self.contact = HIL.BASE.make_contact(349) + + def test_encoder_matches_all_three_firmware_boundaries(self): + with_lastmod = HIL.encode_contact_update(self.contact) + gps_only = HIL.encode_contact_update( + self.contact, include_lastmod=False + ) + self.assertEqual(len(with_lastmod), HIL.CONTACT_UPDATE_LASTMOD_LEN) + self.assertEqual(len(gps_only), HIL.CONTACT_UPDATE_GPS_LEN) + self.assertEqual(with_lastmod[0], HIL.CMD_ADD_UPDATE_CONTACT) + self.assertEqual(with_lastmod[1:33].hex(), self.contact["public_key"]) + self.assertEqual(with_lastmod[33], 1) + self.assertEqual(with_lastmod[35], 0xFF) + + def test_case_matrix_covers_every_requested_forbidden_length(self): + cases = HIL.malformed_cases(self.contact) + lengths = { + len(case.frame) + for case in cases + if case.name.startswith("forbidden_length_") + } + self.assertEqual(lengths, set(HIL.FORBIDDEN_CONTACT_UPDATE_LENGTHS)) + self.assertEqual( + lengths, + {135, *range(137, 144), *range(145, 148)}, + ) + self.assertTrue( + all(case.frame[0] == HIL.CMD_ADD_UPDATE_CONTACT for case in cases) + ) + + def test_invalid_type_and_path_offsets_are_exact(self): + cases = {case.name: case.frame for case in HIL.malformed_cases(self.contact)} + for value in HIL.INVALID_CONTACT_TYPES: + frame = cases[f"invalid_type_{value}"] + self.assertEqual(len(frame), 148) + self.assertEqual(frame[33], value) + for value in HIL.INVALID_PATH_LENGTHS: + frame = cases[f"invalid_path_{value:02x}"] + self.assertEqual(len(frame), 148) + self.assertEqual(frame[35], value) + + def test_illegal_arg_requires_error_code_six(self): + HIL.require_illegal_arg( + SimpleNamespace( + type=SimpleNamespace(name="ERROR"), payload={"error_code": 6} + ), + "case", + ) + for event in ( + SimpleNamespace(type=SimpleNamespace(name="OK"), payload={}), + SimpleNamespace( + type=SimpleNamespace(name="ERROR"), payload={"error_code": 5} + ), + ): + with self.assertRaises(HIL.ProtocolHilFailure): + HIL.require_illegal_arg(event, "case") + + +class SnapshotTests(unittest.TestCase): + def setUp(self): + self.contacts = { + contact["public_key"]: contact + for contact in ( + HIL.BASE.make_contact(index) + for index in range(HIL.EXPECTED_CONTACT_COUNT) + ) + } + + def test_exact_deterministic_table_has_locked_digest(self): + result = HIL.validate_exact_snapshot( + self.contacts, require_canonical_fields=True + ) + self.assertEqual(result["count"], 350) + self.assertEqual(result["keyset_sha256"], HIL.EXPECTED_KEYSET_SHA256) + + def test_missing_key_and_same_keys_changed_content_are_rejected(self): + missing = dict(self.contacts) + missing.pop(next(iter(missing))) + with self.assertRaisesRegex(HIL.ProtocolHilFailure, "expected 350"): + HIL.validate_exact_snapshot(missing, require_canonical_fields=True) + + changed = {key: dict(value) for key, value in self.contacts.items()} + changed[next(iter(changed))]["adv_name"] = "mutated" + with self.assertRaisesRegex(HIL.ProtocolHilFailure, "unexpected contact fields"): + HIL.validate_exact_snapshot(changed, require_canonical_fields=True) + + def test_wire_digest_includes_lastmod_while_canonical_digest_does_not(self): + baseline = {key: dict(value) for key, value in self.contacts.items()} + changed = {key: dict(value) for key, value in baseline.items()} + key = next(iter(changed)) + baseline[key]["lastmod"] = 100 + changed[key]["lastmod"] = 101 + + self.assertEqual( + HIL.semantic_digest(baseline), HIL.semantic_digest(changed) + ) + self.assertNotEqual( + HIL.wire_snapshot_digest(baseline), + HIL.wire_snapshot_digest(changed), + ) + + +def fake_event(name, payload=None): + return SimpleNamespace( + type=SimpleNamespace(name=name), + payload={} if payload is None else payload, + ) + + +class FakeMalformedCommands: + def __init__(self): + self.frames = [] + + async def send(self, frame, expected_events, timeout): + self.frames.append((bytes(frame), tuple(expected_events), timeout)) + return fake_event("ERROR", {"error_code": 6}) + + +class FakeMalformedBase: + def __init__(self, contacts): + self._contacts = contacts + self.mc = SimpleNamespace(commands=FakeMalformedCommands()) + self.contact_reads = 0 + + async def contacts(self): + self.contact_reads += 1 + return {key: dict(value) for key, value in self._contacts.items()} + + +class MalformedExecutionTests(unittest.IsolatedAsyncioTestCase): + async def test_every_rejection_is_followed_by_an_exact_inventory(self): + contacts = { + contact["public_key"]: contact + for contact in ( + HIL.BASE.make_contact(index) + for index in range(HIL.EXPECTED_CONTACT_COUNT) + ) + } + subject = HIL.ProtocolRunner( + valid_config(), SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL) + ) + fake = FakeMalformedBase(contacts) + subject.base = fake + + await subject.test_malformed_updates(contacts) + + self.assertEqual(len(fake.mc.commands.frames), 17) + self.assertEqual(fake.contact_reads, 17) + self.assertEqual(subject.steps[-1]["step"], "malformed_contact_updates") + self.assertTrue(subject.steps[-1]["unchanged_after_each"]) + self.assertTrue( + all(frame[0] == HIL.CMD_ADD_UPDATE_CONTACT for frame, _, _ in fake.mc.commands.frames) + ) + + +class FakeStreamReader: + def __init__(self, owner): + self.owner = owner + self.contact_nb = None + self.contacts = {} + + async def handle_rx(self, raw): + packet_type = raw[0] + if packet_type == HIL.RESP_CODE_CONTACTS_START: + self.contact_nb = int.from_bytes(raw[1:5], "little") + self.contacts = {} + elif packet_type == HIL.RESP_CODE_CONTACT: + key = raw[1:33].hex() + self.contacts[key] = dict(self.owner.table[key]) + + +class FakeStreamCommands: + def __init__(self, owner): + self.owner = owner + + @staticmethod + def _start(count): + return bytes((HIL.RESP_CODE_CONTACTS_START,)) + struct.pack("= deadline: + return fake_event("ERROR", {"reason": "mutation did not run"}) + await asyncio.sleep(0) + + # Both local mutation commands complete before this task is scheduled + # again, representing the firmware's revision-triggered fresh snapshot. + await self.owner.reader.handle_rx(self._start(len(self.owner.table))) + for key in sorted(self.owner.table): + await self.owner.reader.handle_rx(self._contact(key)) + await asyncio.sleep(0) + await self.owner.reader.handle_rx( + bytes((HIL.RESP_CODE_END_OF_CONTACTS,)) + b"\0" * 4 + ) + return fake_event("CONTACTS", dict(self.owner.reader.contacts)) + + async def remove_contact(self, key): + self.owner.table.pop(key) + self.owner.revision += 1 + return fake_event("OK") + + async def add_contact(self, contact): + self.owner.table[contact["public_key"]] = dict(contact) + self.owner.revision += 1 + return fake_event("OK") + + async def send(self, frame, expected_events, timeout): + self.owner.table[self.owner.original_target["public_key"]] = dict( + self.owner.original_target + ) + self.owner.revision += 1 + return fake_event("OK") + + +class FakeStreamMesh: + def __init__(self, contacts): + self.table = {key: dict(value) for key, value in contacts.items()} + target_key = HIL.BASE.make_contact(349)["public_key"] + self.original_target = dict(self.table[target_key]) + self.revision = 0 + self.reader = FakeStreamReader(self) + self._reader = self.reader + self.commands = FakeStreamCommands(self) + + +class StructuralExecutionTests(unittest.IsolatedAsyncioTestCase): + async def test_live_remove_readd_requires_raw_restart_and_exact_sdk_result(self): + contacts = { + contact["public_key"]: contact + for contact in ( + HIL.BASE.make_contact(index) + for index in range(HIL.EXPECTED_CONTACT_COUNT) + ) + } + subject = HIL.ProtocolRunner( + valid_config(), SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL) + ) + subject.base.mc = FakeStreamMesh(contacts) + target_key = HIL.BASE.make_contact(349)["public_key"] + subject.original_target_contact = dict(contacts[target_key]) + + await subject.test_structural_restart() + + step = subject.steps[-1] + self.assertEqual(step["step"], "structural_stream_restart") + self.assertGreaterEqual(step["raw"]["fresh_starts"], 1) + self.assertEqual(step["raw"]["completed_count"], 350) + self.assertEqual(step["sdk"]["keyset_sha256"], HIL.EXPECTED_KEYSET_SHA256) + self.assertEqual(subject.base.mc.revision, 2) + + +class FakeRestoreCommands: + def __init__(self, owner): + self.owner = owner + self.sent_frames = [] + + async def send(self, frame, expected_events, timeout): + self.sent_frames.append(bytes(frame)) + self.owner.table[self.owner.target["public_key"]] = dict(self.owner.target) + return fake_event("OK") + + async def get_autoadd_config(self): + return fake_event("AUTOADD_CONFIG", {"config": self.owner.autoadd}) + + async def send_appstart(self): + return fake_event( + "SELF_INFO", {"manual_add_contacts": self.owner.manual} + ) + + +class FakeRestoreBase: + def __init__(self, table, target): + self.table = {key: dict(value) for key, value in table.items()} + self.target = dict(target) + self.autoadd = 0 + self.manual = True + self.reboots = 0 + self.mc = SimpleNamespace(commands=FakeRestoreCommands(self)) + + @staticmethod + def _require_event(event, expected): + return HIL.require_event(event, expected, "fake") + + async def contacts(self): + return {key: dict(value) for key, value in self.table.items()} + + async def reboot_and_reconnect(self): + self.reboots += 1 + + async def set_autoadd_config(self, value): + self.autoadd = value + + async def set_manual_contact_mode(self, value): + self.manual = value + + +class RestorationTests(unittest.IsolatedAsyncioTestCase): + async def test_restoration_preserves_lastmod_reboots_and_restores_admission(self): + table = {} + for index in range(HIL.EXPECTED_CONTACT_COUNT): + contact = HIL.BASE.make_contact(index) + contact["lastmod"] = 10_000 + index + table[contact["public_key"]] = contact + target_key = HIL.BASE.make_contact(349)["public_key"] + target = dict(table[target_key]) + table[target_key] = {**target, "lastmod": 999_999} + + subject = HIL.ProtocolRunner( + valid_config(settle_seconds=0.001), + SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL), + ) + fake = FakeRestoreBase(table, target) + subject.base = fake + subject.restoration_armed = True + subject.original_target_contact = target + expected_table = {key: dict(value) for key, value in table.items()} + expected_table[target_key] = target + subject.baseline_wire_sha256 = HIL.wire_snapshot_digest(expected_table) + subject.original_autoadd_config = 31 + subject.original_manual_add_contacts = False + + await subject.restore() + + self.assertEqual(fake.reboots, 1) + self.assertEqual(fake.table[target_key]["lastmod"], target["lastmod"]) + self.assertEqual(len(fake.mc.commands.sent_frames), 1) + self.assertEqual(len(fake.mc.commands.sent_frames[0]), 148) + self.assertEqual(fake.autoadd, 31) + self.assertFalse(fake.manual) + self.assertEqual( + [step["step"] for step in subject.steps[-3:]], + [ + "contact_table_restored", + "autoadd_config_restored", + "contact_admission_restored", + ], + ) + + +class RawContactStreamTests(unittest.TestCase): + @staticmethod + def start(count): + return bytes((HIL.RESP_CODE_CONTACTS_START,)) + struct.pack(" str: + return (bytes([byte]) * 32).hex() + + +def event(name: str, payload=None): + return SimpleNamespace( + type=SimpleNamespace(name=name), payload={} if payload is None else payload + ) + + +class FakeCommands: + def __init__(self, owner, responses): + self.owner = owner + self.responses = list(responses) + self.calls = [] + + async def get_contacts(self, *, lastmod, timeout): + self.calls.append((lastmod, timeout)) + contacts, advertised = self.responses.pop(0) + self.owner._reader.contacts = dict(contacts) + if advertised is not None: + self.owner._reader.contact_nb = advertised + return SimpleNamespace( + type=SimpleNamespace(name="CONTACTS"), payload=dict(contacts) + ) + + +class FakeMeshCore: + def __init__(self, responses): + self._reader = SimpleNamespace(contact_nb=999, contacts={"stale": {}}) + self.commands = FakeCommands(self, responses) + + +def runner(responses, *, passes=None, digest=None): + config = SimpleNamespace( + contact_enumeration_passes=passes, + expected_contact_keyset_sha256=digest, + verbose=False, + ) + identity = SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL) + result = HIL.StressRunner(config, identity) + result.mc = FakeMeshCore(responses) + return result + + +class ContactEnumerationHarnessTests(unittest.IsolatedAsyncioTestCase): + async def test_safe_snapshot_retries_a_short_stream(self): + expected = {key(1): {}, key(2): {}} + subject = runner([({key(1): {}}, 2), (expected, 2)]) + + actual = await subject.contacts() + + self.assertEqual(set(actual), set(expected)) + self.assertEqual(subject.contact_enumeration_transactions, 2) + self.assertEqual(subject.contact_enumeration_short_attempts, 1) + self.assertEqual(subject.contact_enumeration_retries, 1) + self.assertEqual(subject.mc.commands.calls, [(0, 20), (0, 20)]) + + async def test_safe_snapshot_never_returns_three_short_streams(self): + subject = runner([({key(1): {}}, 2)] * 3) + + with self.assertRaisesRegex( + HIL.HilFailure, r"incomplete contact enumeration; attempt 1:" + ): + await subject.contacts() + + self.assertEqual(subject.contact_enumeration_transactions, 3) + self.assertEqual(subject.contact_enumeration_short_attempts, 3) + self.assertEqual(subject.contact_enumeration_retries, 2) + + async def test_missing_start_count_cannot_reuse_stale_reader_state(self): + subject = runner([({key(1): {}}, None)]) + + with self.assertRaisesRegex(HIL.HilFailure, "valid CONTACTS_START count"): + await subject.contacts() + + self.assertIsNone(subject.mc._reader.contact_nb) + + async def test_repeated_stress_accepts_only_the_same_complete_keyset(self): + expected = {key(1), key(2), key(3)} + payload = {item: {} for item in expected} + subject = runner([(payload, 3)] * 4, passes=4) + + actual = await subject.stress_contact_enumerations( + expected, "unit-test table" + ) + + self.assertEqual(set(actual), expected) + self.assertEqual(len(subject.mc.commands.calls), 4) + step = subject.steps[-1] + self.assertEqual(step["step"], "contact_enumeration_stress") + self.assertTrue(step["ok"]) + self.assertEqual(step["completed_passes"], 4) + self.assertEqual(step["transport_transactions"], 4) + self.assertEqual(step["short_attempts"], 0) + self.assertEqual(step["retry_attempts"], 0) + self.assertEqual(step["advertised_counts"], [3, 3, 3, 3]) + self.assertEqual(step["received_counts"], [3, 3, 3, 3]) + + async def test_stress_reports_retried_short_stream_and_fails(self): + expected = {key(1), key(2)} + complete = {item: {} for item in expected} + subject = runner( + [({key(1): {}}, 2), (complete, 2), (complete, 2)], passes=2 + ) + + with self.assertRaisesRegex( + HIL.HilFailure, + r"observed 1 incomplete contact stream\(s\).*1 retry attempt", + ): + await subject.stress_contact_enumerations(expected, "unit-test table") + + step = subject.steps[-1] + self.assertFalse(step["ok"]) + self.assertEqual(step["completed_passes"], 2) + self.assertEqual(step["transport_transactions"], 3) + self.assertEqual(step["short_attempts"], 1) + self.assertEqual(step["retry_attempts"], 1) + self.assertEqual(step["advertised_counts"], [2, 2]) + self.assertEqual(step["received_counts"], [2, 2]) + + async def test_same_count_wrong_keyset_fails_on_exact_pass(self): + expected = {key(1), key(2)} + wrong = {key(1): {}, key(3): {}} + subject = runner([(wrong, 2)], passes=1) + + with self.assertRaisesRegex( + HIL.HilFailure, r"pass 1/1: contact key set mismatch" + ): + await subject.stress_contact_enumerations(expected, "unit-test table") + + self.assertEqual(subject.steps[-1]["completed_passes"], 0) + self.assertEqual(len(subject.mc.commands.calls), 1) + + async def test_expected_keyset_digest_anchors_read_only_stress(self): + expected = {key(1), key(2)} + payload = {item: {} for item in expected} + subject = runner([(payload, 2)], passes=1, digest="0" * 64) + + with self.assertRaisesRegex(HIL.HilFailure, "keyset digest is"): + await subject.stress_contact_enumerations(None, "unit-test table") + + self.assertFalse(subject.steps[-1]["ok"]) + + +class ContactEnumerationArgumentTests(unittest.TestCase): + def test_new_corruption_option_is_backward_compatible_and_off_by_default(self): + legacy_positional = HIL.RunConfig( + "COM23", + "inspect", + False, + None, + None, + 350, + 131072, + 39, + 7, + 35, + False, + False, + None, + False, + None, + None, + False, + ) + self.assertIsNone(legacy_positional.corrupt_occupied_page) + self.assertFalse(legacy_positional.advert_remove_rollback) + self.assertIsNone(legacy_positional.fail_read_page) + self.assertIsNone(legacy_positional.fail_stat_page) + + args = HIL.build_argument_parser().parse_args(["--port", "COM23"]) + del args.corrupt_occupied_page + del args.advert_remove_rollback + del args.fail_read_page + del args.fail_stat_page + config = HIL.config_from_args(args) + self.assertIsNone(config.corrupt_occupied_page) + self.assertFalse(config.advert_remove_rollback) + self.assertIsNone(config.fail_read_page) + self.assertIsNone(config.fail_stat_page) + + def test_parser_accepts_read_only_stress_without_destructive_consent(self): + args = HIL.build_argument_parser().parse_args( + [ + "--port", + "COM23", + "--contact-enumeration-passes", + "25", + "--expected-contact-keyset-sha256", + "AB" * 32, + ] + ) + + config = HIL.config_from_args(args) + + self.assertEqual(config.scenario, "inspect") + self.assertEqual(config.contact_enumeration_passes, 25) + self.assertEqual(config.expected_contact_keyset_sha256, "ab" * 32) + HIL.require_destructive_consent(config) + + def test_digest_requires_enumeration_option(self): + args = HIL.build_argument_parser().parse_args( + [ + "--port", + "COM23", + "--expected-contact-keyset-sha256", + "ab" * 32, + ] + ) + + with self.assertRaisesRegex( + HIL.HilFailure, "requires --contact-enumeration-passes" + ): + HIL.config_from_args(args) + + def test_runtime_port_gate_still_rejects_wrong_physical_identity(self): + wrong = SimpleNamespace( + device="COM23", + serial_number="NOT-THE-SPARE", + vid=HIL.EXPECTED_USB_VID, + pid=HIL.EXPECTED_USB_PID, + description="T1000-E", + product="T1000-E", + interface="MI_00", + location="1-8:x.0", + hwid="USB MI_00", + ) + + with self.assertRaisesRegex(HIL.HilFailure, "expected the locked spare"): + HIL.resolve_initial_port("COM23", port_source=lambda: [wrong]) + + def test_occupied_mask_corruption_is_comprehensive_only_and_consent_gated(self): + parser = HIL.build_argument_parser() + inspect_args = parser.parse_args( + ["--port", "COM23", "--corrupt-occupied-page", "13"] + ) + with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"): + HIL.config_from_args(inspect_args) + + comprehensive_args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--corrupt-occupied-page", + "13", + "--allow-existing-contacts", + ] + ) + config = HIL.config_from_args(comprehensive_args) + self.assertEqual(config.corrupt_occupied_page, 13) + with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"): + HIL.require_destructive_consent(config) + + def test_restorative_page_corruption_accepts_reviewed_existing_inventory(self): + parser = HIL.build_argument_parser() + args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--corrupt-page", + "7", + "--allow-existing-contacts", + "--allow-destructive", + "--confirm-usb-serial", + HIL.EXPECTED_USB_SERIAL, + ] + ) + + config = HIL.config_from_args(args) + + self.assertEqual(config.corrupt_page, 7) + self.assertTrue(config.allow_existing_contacts) + HIL.require_destructive_consent(config) + + def test_contact_page_mask_parser_is_strict(self): + self.assertEqual( + HIL.parse_contact_page_mask( + "HIL contact page 13 occupied=81Ff00a5" + ), + HIL.ContactPageMask(13, 0x81FF00A5), + ) + for malformed in ( + "HIL contact page 14 occupied=01ffffff", + "HIL contact page 0 occupied=1ffffff", + "HIL contact page 0 mask=01ffffff", + ): + with self.subTest(malformed=malformed): + with self.assertRaises(HIL.HilFailure): + HIL.parse_contact_page_mask(malformed) + + def test_advert_remove_rollback_is_opt_in_comprehensive_and_gated(self): + parser = HIL.build_argument_parser() + inspect_args = parser.parse_args( + ["--port", "COM23", "--advert-remove-rollback"] + ) + with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"): + HIL.config_from_args(inspect_args) + + args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--advert-remove-rollback", + ] + ) + config = HIL.config_from_args(args) + self.assertTrue(config.advert_remove_rollback) + with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"): + HIL.require_destructive_consent(config) + + def test_contact_slot_and_advert_seed_parsers_are_strict(self): + self.assertEqual(HIL.parse_contact_slot("HIL contact slot=349"), 349) + self.assertEqual(HIL.parse_advert_seed_slot("HIL advert seeded slot=0"), 0) + for parser, malformed in ( + (HIL.parse_contact_slot, "HIL contact slot=350"), + (HIL.parse_contact_slot, "HIL contact slot=-1"), + (HIL.parse_advert_seed_slot, "HIL advert seed failed"), + (HIL.parse_advert_seed_slot, "HIL advert seeded slot=999"), + ): + with self.subTest(malformed=malformed): + with self.assertRaises(HIL.HilFailure): + parser(malformed) + + def test_fail_read_page_is_bounded_comprehensive_only_and_gated(self): + parser = HIL.build_argument_parser() + inspect_args = parser.parse_args( + ["--port", "COM23", "--fail-read-page", "13"] + ) + with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"): + HIL.config_from_args(inspect_args) + + with self.assertRaisesRegex(argparse.ArgumentTypeError, "in 0..13"): + HIL._nonnegative_page("14") + + args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--fail-read-page", + "0", + ] + ) + config = HIL.config_from_args(args) + self.assertEqual(config.fail_read_page, 0) + with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"): + HIL.require_destructive_consent(config) + + def test_fail_stat_page_is_bounded_exclusive_and_gated(self): + parser = HIL.build_argument_parser() + inspect_args = parser.parse_args( + ["--port", "COM23", "--fail-stat-page", "13"] + ) + with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"): + HIL.config_from_args(inspect_args) + + args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--fail-stat-page", + "0", + ] + ) + config = HIL.config_from_args(args) + self.assertEqual(config.fail_stat_page, 0) + with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"): + HIL.require_destructive_consent(config) + + both = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--fail-read-page", + "0", + "--fail-stat-page", + "1", + ] + ) + with self.assertRaisesRegex(HIL.HilFailure, "mutually exclusive"): + HIL.config_from_args(both) + + +class ExactContactFrameTests(unittest.TestCase): + def test_exact_frame_preserves_lastmod_and_all_contact_fields(self): + contact = HIL.make_contact(7) + contact.update( + { + "flags": 5, + "out_path_hash_mode": 1, + "out_path_len": 2, + "out_path": "00112233", + "adv_name": "HIL-EXACT", + "adv_lat": 12.345678, + "adv_lon": -87.654321, + "lastmod": 1_800_000_007, + } + ) + + frame = HIL.encode_exact_contact_frame(contact) + + self.assertEqual(len(frame), 148) + self.assertEqual(frame[0], 0x09) + self.assertEqual(frame[1:33], bytes.fromhex(contact["public_key"])) + self.assertEqual(frame[33:36], bytes((1, 5, 0x42))) + self.assertEqual(frame[36:40], bytes.fromhex("00112233")) + self.assertEqual(frame[40:100], bytes(60)) + self.assertEqual(frame[100:109], b"HIL-EXACT") + self.assertEqual(frame[109:132], bytes(23)) + self.assertEqual( + int.from_bytes(frame[132:136], "little"), contact["last_advert"] + ) + self.assertEqual( + int.from_bytes(frame[136:140], "little", signed=True), 12_345_678 + ) + self.assertEqual( + int.from_bytes(frame[140:144], "little", signed=True), -87_654_321 + ) + self.assertEqual(int.from_bytes(frame[144:148], "little"), 1_800_000_007) + + def test_exact_frame_requires_original_lastmod(self): + with self.assertRaisesRegex(HIL.HilFailure, "lastmod"): + HIL.encode_exact_contact_frame(HIL.make_contact(7)) + + +class DestructiveScenarioTests(unittest.IsolatedAsyncioTestCase): + def make_runner(self): + config = SimpleNamespace( + contact_count=350, + contact_enumeration_passes=None, + expected_contact_keyset_sha256=None, + settle_seconds=0.001, + corrupt_occupied_page=0, + allow_existing_contacts=True, + verbose=False, + ) + identity = SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL) + return HIL.StressRunner(config, identity) + + @staticmethod + def full_contacts(): + contacts = [HIL.make_contact(index) for index in range(350)] + for index, contact in enumerate(contacts): + contact["lastmod"] = 1_800_000_000 + index + return {contact["public_key"]: dict(contact) for contact in contacts} + + async def test_occupied_mask_scenario_repairs_populated_empty_and_high_bits(self): + subject = self.make_runner() + contacts = self.full_contacts() + target_key = HIL.make_contact(0)["public_key"] + without_target = { + key_value: value + for key_value, value in contacts.items() + if key_value != target_key + } + subject.contacts = AsyncMock( + side_effect=[ + contacts, + contacts, + contacts, + contacts, + without_target, + without_target, + without_target, + without_target, + contacts, + ] + ) + subject.reboot_and_reconnect = AsyncMock() + subject.cli = AsyncMock( + side_effect=[ + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupancy bit 0 corrupted", + "HIL contact page 0 occupied=01fffffe", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupancy bit 31 corrupted", + "HIL contact page 0 occupied=81ffffff", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01fffffe", + "HIL contact page 0 occupancy bit 0 corrupted", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01fffffe", + "HIL contact page 0 occupied=01fffffe", + "HIL contact page 0 occupied=01ffffff", + ] + ) + commands = SimpleNamespace( + remove_contact=AsyncMock(return_value=event("OK")), + add_contact=AsyncMock(return_value=event("OK")), + update_contact=AsyncMock(return_value=event("OK")), + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_contact_occupied_mask_recovery(0, contacts) + + self.assertEqual(subject.reboot_and_reconnect.await_count, 8) + self.assertEqual(subject.contacts.await_count, 9) + commands.remove_contact.assert_awaited_once_with(target_key) + commands.add_contact.assert_awaited_once_with(contacts[target_key]) + commands.update_contact.assert_not_awaited() + self.assertEqual( + [call.args[0] for call in subject.cli.await_args_list], + [ + "hil extrafs page-mask 0", + "hil extrafs corrupt-occupied 0 0 CONFIRM", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs corrupt-occupied 0 31 CONFIRM", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs corrupt-occupied 0 0 CONFIRM", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + "hil extrafs page-mask 0", + ], + ) + step = subject.steps[-1] + self.assertEqual(step["step"], "contact_occupied_mask_recovery") + self.assertEqual(step["contacts_preserved"], 350) + self.assertEqual( + [case["corrupted_mask"] for case in step["cases"]], + ["01fffffe", "81ffffff", "01ffffff"], + ) + self.assertEqual( + [case["durable_mask"] for case in step["cases"]], + ["01ffffff", "01ffffff", "01fffffe"], + ) + empty_case = step["cases"][-1] + self.assertEqual(empty_case["kind"], "valid empty") + self.assertEqual(empty_case["contact_count"], 349) + self.assertEqual(empty_case["restored_contact_count"], 350) + self.assertEqual(empty_case["restored_mask"], "01ffffff") + self.assertTrue(step["original_contact_restored"]) + + async def test_crc_restore_adds_missing_page_in_original_slot_order(self): + subject = self.make_runner() + baseline = self.full_contacts() + baseline_order = list(baseline) + expected_slots = { + public_key: slot for slot, public_key in enumerate(baseline_order) + } + page = 7 + first = page * HIL.EXPECTED_CONTACTS_PER_PAGE + lost = set(baseline_order[first:first + HIL.EXPECTED_CONTACTS_PER_PAGE]) + surviving = { + public_key: value + for public_key, value in baseline.items() + if public_key not in lost + } + subject.contacts = AsyncMock(side_effect=[baseline, baseline]) + subject.assert_contact_slot_map = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + subject.contact_slot = AsyncMock( + side_effect=list( + range(first, first + HIL.EXPECTED_CONTACTS_PER_PAGE) + ) + ) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.reboot_and_reconnect = AsyncMock() + + restored, digests = await subject.restore_contact_inventory_exact( + baseline, + baseline_order, + expected_slots, + page, + current=surviving, + context="unit CRC restore", + ) + + self.assertEqual(restored, baseline) + self.assertEqual(digests, subject.exact_inventory_digests(baseline)) + self.assertEqual( + [ + call.args[0]["public_key"] + for call in subject.set_contact_exact.await_args_list + ], + baseline_order[first:first + HIL.EXPECTED_CONTACTS_PER_PAGE], + ) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual(subject.contacts.await_count, 2) + self.assertEqual(subject.assert_contact_slot_map.await_count, 2) + self.assertEqual(subject.contact_page_mask.await_count, 3) + + async def test_crc_page_corruption_restores_exact_inventory_and_slots(self): + subject = self.make_runner() + subject.config.expected_contact_keyset_sha256 = None + baseline = self.full_contacts() + baseline_order = list(baseline) + expected_slots = { + public_key: slot for slot, public_key in enumerate(baseline_order) + } + page = 7 + first = page * HIL.EXPECTED_CONTACTS_PER_PAGE + lost_order = baseline_order[ + first:first + HIL.EXPECTED_CONTACTS_PER_PAGE + ] + surviving = { + public_key: value + for public_key, value in baseline.items() + if public_key not in set(lost_order) + } + status = HIL.ExtraFsStatus(True, 60, 100, 0) + digests = subject.exact_inventory_digests(baseline) + subject.clear_filler = AsyncMock() + subject.status = AsyncMock(side_effect=[status, status]) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.capture_contact_slot_map = AsyncMock( + return_value=expected_slots + ) + subject.cli = AsyncMock( + return_value=f"HIL contact page {page} CRC corrupted" + ) + subject.reboot_and_reconnect = AsyncMock() + subject.contacts = AsyncMock(return_value=surviving) + subject.expect_contact_not_found = AsyncMock() + subject.restore_contact_inventory_exact = AsyncMock( + return_value=(baseline, digests) + ) + + result = await subject.corrupt_contact_page(page, baseline) + + self.assertEqual(result, set(baseline)) + subject.cli.assert_awaited_once_with( + f"hil extrafs corrupt-page {page} CONFIRM" + ) + self.assertEqual(subject.reboot_and_reconnect.await_count, 1) + self.assertEqual(subject.expect_contact_not_found.await_count, 25) + self.assertEqual( + [call.args[0] for call in subject.expect_contact_not_found.await_args_list], + lost_order, + ) + restore_call = subject.restore_contact_inventory_exact.await_args + self.assertEqual(restore_call.args[0], baseline) + self.assertEqual(restore_call.args[1], baseline_order) + self.assertEqual(restore_call.args[2], expected_slots) + self.assertEqual(restore_call.args[3], page) + self.assertEqual(restore_call.kwargs["current"], surviving) + step = subject.steps[-1] + self.assertEqual(step["step"], "contact_page_crc_loss_and_restore") + self.assertEqual(step["isolated_after"], 325) + self.assertEqual(step["lost_slots"], list(range(175, 200))) + self.assertTrue(step["slots_restored"]) + self.assertTrue(step["stream_order_restored"]) + self.assertEqual(step["semantic_sha256"], digests["semantic_sha256"]) + self.assertEqual(step["wire_sha256"], digests["wire_sha256"]) + + async def test_crc_page_failure_runs_emergency_exact_restore(self): + subject = self.make_runner() + subject.config.expected_contact_keyset_sha256 = None + baseline = self.full_contacts() + baseline_order = list(baseline) + expected_slots = { + public_key: slot for slot, public_key in enumerate(baseline_order) + } + page = 7 + lost = set(baseline_order[175:200]) + # An extra missing record proves the isolation assertion fails after + # corruption, while the emergency path still restores the baseline. + lost.add(baseline_order[200]) + wrong_survivors = { + public_key: value + for public_key, value in baseline.items() + if public_key not in lost + } + status = HIL.ExtraFsStatus(True, 60, 100, 0) + digests = subject.exact_inventory_digests(baseline) + subject.clear_filler = AsyncMock() + subject.status = AsyncMock(side_effect=[status, status]) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.capture_contact_slot_map = AsyncMock( + return_value=expected_slots + ) + subject.cli = AsyncMock( + return_value=f"HIL contact page {page} CRC corrupted" + ) + subject.reboot_and_reconnect = AsyncMock() + subject.contacts = AsyncMock(return_value=wrong_survivors) + subject.restore_contact_inventory_exact = AsyncMock( + return_value=(baseline, digests) + ) + + with self.assertRaisesRegex(HIL.HilFailure, "contact key set mismatch"): + await subject.corrupt_contact_page(page, baseline) + + self.assertEqual(subject.clear_filler.await_count, 2) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + emergency_call = subject.restore_contact_inventory_exact.await_args + self.assertNotIn("current", emergency_call.kwargs) + self.assertEqual( + emergency_call.kwargs["context"], + "emergency contact-page CRC restoration", + ) + step = subject.steps[-1] + self.assertEqual( + step["step"], "contact_page_corruption_emergency_restore" + ) + self.assertTrue(step["recovered_exactly"]) + self.assertEqual(step["wire_sha256"], digests["wire_sha256"]) + + async def test_empty_mask_failure_emergency_restores_exact_contact(self): + subject = self.make_runner() + contacts = self.full_contacts() + target_key = HIL.make_contact(0)["public_key"] + without_target = { + key_value: value + for key_value, value in contacts.items() + if key_value != target_key + } + subject.contacts = AsyncMock( + side_effect=[ + contacts, + contacts, + contacts, + contacts, + without_target, + without_target, + contacts, + ] + ) + subject.reboot_and_reconnect = AsyncMock( + side_effect=[None, None, None, None, None, HIL.HilFailure("injected"), None] + ) + subject.cli = AsyncMock( + side_effect=[ + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupancy bit 0 corrupted", + "HIL contact page 0 occupied=01fffffe", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupancy bit 31 corrupted", + "HIL contact page 0 occupied=81ffffff", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01fffffe", + "HIL contact page 0 occupancy bit 0 corrupted", + "HIL contact page 0 occupied=01ffffff", + "HIL contact page 0 occupied=01ffffff", + ] + ) + commands = SimpleNamespace( + remove_contact=AsyncMock(return_value=event("OK")), + add_contact=AsyncMock(return_value=event("OK")), + update_contact=AsyncMock(return_value=event("OK")), + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "injected"): + await subject.test_contact_occupied_mask_recovery(0, contacts) + + commands.remove_contact.assert_awaited_once_with(target_key) + commands.add_contact.assert_awaited_once_with(contacts[target_key]) + commands.update_contact.assert_not_awaited() + self.assertEqual(subject.contacts.await_count, 7) + self.assertEqual(subject.reboot_and_reconnect.await_count, 7) + self.assertEqual( + subject.steps[-1]["step"], + "contact_occupied_mask_emergency_restore", + ) + self.assertTrue(subject.steps[-1]["original_restored"]) + + async def test_occupied_mask_preflight_rejects_non_harness_inventory(self): + subject = self.make_runner() + subject.contacts = AsyncMock( + return_value={key(0xAA): {"public_key": key(0xAA)}} + ) + + with self.assertRaisesRegex( + HIL.HilFailure, "inventory made only from this harness" + ): + await subject.add_contacts_to_capacity() + + async def test_storage_fill_helper_is_bounded_verified_and_scoped(self): + subject = self.make_runner() + subject.config.fill_bytes = 131072 + subject.clear_filler = AsyncMock() + subject.cli = AsyncMock( + return_value=( + "HIL ExtraFS fill requested=131072 written=36864 " + "used=99KiB total=100KiB" + ) + ) + + fill = await subject.fill_storage_to_enospc() + + subject.clear_filler.assert_awaited_once() + subject.cli.assert_awaited_once_with( + "hil extrafs fill 131072", + timeout=HIL.EXTRAFS_FILL_TIMEOUT_SECONDS, + ) + self.assertEqual(fill, HIL.ExtraFsFill(131072, 36864, 99, 100)) + + async def test_full_storage_contact_ack_refusal_recovery_and_restore(self): + subject = self.make_runner() + original = self.full_contacts() + target_key = HIL.make_contact(349)["public_key"] + changed = {key_value: dict(value) for key_value, value in original.items()} + changed[target_key]["adv_name"] = "HIL-ENOSPC" + changed[target_key]["flags"] = 1 + subject.contacts = AsyncMock( + side_effect=[original, changed, changed, changed, original] + ) + subject.generated_keys = list(original) + subject.core_uptime = AsyncMock(side_effect=[20, 21]) + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + commands = SimpleNamespace( + update_contact=AsyncMock(side_effect=[event("OK"), event("OK")]), + send=AsyncMock(return_value=event("ERROR", {"error_code": 5})), + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_contact_update_recovery_when_full(set(original)) + + self.assertEqual(commands.update_contact.await_count, 2) + first_update = commands.update_contact.await_args_list[0].args[0] + second_update = commands.update_contact.await_args_list[1].args[0] + self.assertEqual(first_update["adv_name"], "HIL-ENOSPC") + self.assertEqual(first_update["flags"], 1) + self.assertEqual(second_update["adv_name"], "HIL-349") + self.assertEqual(second_update["flags"], 0) + reboot_call = commands.send.await_args + self.assertEqual(reboot_call.args[0], b"\x13reboot") + self.assertEqual( + reboot_call.kwargs["timeout"], + HIL.ENOSPC_REBOOT_REFUSAL_TIMEOUT_SECONDS, + ) + self.assertEqual(subject.clear_filler.await_count, 2) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + step = subject.steps[-1] + self.assertEqual(step["step"], "full_storage_contact_recovery") + self.assertEqual(step["uptime_before_refusal"], 20) + self.assertEqual(step["uptime_after_refusal"], 21) + self.assertTrue(step["persisted_after_filler_clear"]) + self.assertTrue(step["original_restored"]) + + async def test_full_storage_failure_restores_only_harness_owned_contact(self): + subject = self.make_runner() + synthetic = [HIL.make_contact(index) for index in range(349)] + baseline_key = key(0xFF) + mixed = { + **{contact["public_key"]: dict(contact) for contact in synthetic}, + baseline_key: { + **HIL.make_contact(999), + "public_key": baseline_key, + "adv_name": "USER-CONTACT", + }, + } + subject.generated_keys = [contact["public_key"] for contact in synthetic] + subject.contacts = AsyncMock( + side_effect=[mixed, HIL.HilFailure("injected after ACK"), mixed] + ) + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + commands = SimpleNamespace( + update_contact=AsyncMock(side_effect=[event("OK"), event("OK")]), + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "injected after ACK"): + await subject.test_contact_update_recovery_when_full(set(mixed)) + + first_target = commands.update_contact.await_args_list[0].args[0] + restored_target = commands.update_contact.await_args_list[1].args[0] + self.assertNotEqual(first_target["public_key"], baseline_key) + self.assertEqual(restored_target["public_key"], first_target["public_key"]) + self.assertEqual(restored_target["adv_name"], "HIL-348") + subject.clear_filler.assert_awaited_once() + subject.reboot_and_reconnect.assert_awaited_once() + self.assertEqual( + subject.steps[-1]["step"], "full_storage_contact_emergency_restore" + ) + + async def test_full_storage_add_ack_refusal_persistence_and_exact_restore(self): + subject = self.make_runner() + original = self.full_contacts() + original_key = HIL.make_contact(349)["public_key"] + replacement = HIL.make_contact(2351) + replacement_key = replacement["public_key"] + without_original = { + key_value: dict(value) + for key_value, value in original.items() + if key_value != original_key + } + substituted = { + **without_original, + replacement_key: dict(replacement), + } + subject.generated_keys = list(original) + subject.contacts = AsyncMock( + side_effect=[ + original, + without_original, + substituted, + substituted, + substituted, + substituted, + original, + ] + ) + fill = HIL.ExtraFsFill(131072, 36864, 99, 100) + subject.fill_storage_to_enospc = AsyncMock(return_value=fill) + subject.core_uptime = AsyncMock(side_effect=[20, 21]) + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + commands = SimpleNamespace( + remove_contact=AsyncMock(side_effect=[event("OK"), event("OK")]), + add_contact=AsyncMock(side_effect=[event("OK"), event("OK")]), + update_contact=AsyncMock(return_value=event("OK")), + send=AsyncMock(return_value=event("ERROR", {"error_code": 5})), + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_contact_add_recovery_when_full(set(original)) + + self.assertEqual( + [entry.args[0] for entry in commands.remove_contact.await_args_list], + [original_key, replacement_key], + ) + self.assertEqual( + [ + entry.args[0]["public_key"] + for entry in commands.add_contact.await_args_list + ], + [replacement_key], + ) + commands.update_contact.assert_not_awaited() + self.assertEqual( + subject.set_contact_exact.await_args.args[0]["public_key"], + original_key, + ) + self.assertEqual(subject.fill_storage_to_enospc.await_count, 1) + self.assertEqual(subject.clear_filler.await_count, 2) + self.assertEqual(subject.reboot_and_reconnect.await_count, 3) + self.assertEqual(subject.contacts.await_count, 7) + step = subject.steps[-1] + self.assertEqual(step["step"], "full_storage_contact_add_recovery") + self.assertEqual(step["uptime_before_refusal"], 20) + self.assertEqual(step["uptime_after_refusal"], 21) + self.assertTrue(step["add_acknowledged"]) + self.assertTrue(step["persisted_after_filler_clear"]) + self.assertTrue(step["original_restored"]) + self.assertEqual(step["keyset_sha256"], subject.keyset_digest(original)) + + async def test_full_storage_add_failure_emergency_restores_substitution(self): + subject = self.make_runner() + original = self.full_contacts() + original_key = HIL.make_contact(349)["public_key"] + replacement = HIL.make_contact(2351) + replacement_key = replacement["public_key"] + without_original = { + key_value: dict(value) + for key_value, value in original.items() + if key_value != original_key + } + substituted = { + **without_original, + replacement_key: dict(replacement), + } + subject.generated_keys = list(original) + subject.contacts = AsyncMock( + side_effect=[ + original, + without_original, + HIL.HilFailure("injected after ADD ACK"), + substituted, + original, + ] + ) + subject.fill_storage_to_enospc = AsyncMock( + return_value=HIL.ExtraFsFill(131072, 36864, 99, 100) + ) + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + commands = SimpleNamespace( + remove_contact=AsyncMock(side_effect=[event("OK"), event("OK")]), + add_contact=AsyncMock(side_effect=[event("OK"), event("OK")]), + update_contact=AsyncMock(return_value=event("OK")), + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "injected after ADD ACK"): + await subject.test_contact_add_recovery_when_full(set(original)) + + self.assertEqual( + [entry.args[0] for entry in commands.remove_contact.await_args_list], + [original_key, replacement_key], + ) + self.assertEqual( + [ + entry.args[0]["public_key"] + for entry in commands.add_contact.await_args_list + ], + [replacement_key], + ) + commands.update_contact.assert_not_awaited() + self.assertEqual( + subject.set_contact_exact.await_args.args[0]["public_key"], + original_key, + ) + self.assertEqual(subject.clear_filler.await_count, 1) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual( + subject.steps[-1]["step"], + "full_storage_contact_add_emergency_restore", + ) + self.assertEqual( + subject.steps[-1]["keyset_sha256"], subject.keyset_digest(original) + ) + + async def test_full_storage_remove_ack_refusal_persistence_and_exact_restore(self): + subject = self.make_runner() + original = self.full_contacts() + target_key = HIL.make_contact(348)["public_key"] + without_target = { + key_value: dict(value) + for key_value, value in original.items() + if key_value != target_key + } + subject.generated_keys = list(original) + subject.contacts = AsyncMock( + side_effect=[ + original, + without_target, + without_target, + without_target, + without_target, + original, + ] + ) + fill = HIL.ExtraFsFill(131072, 36864, 99, 100) + subject.fill_storage_to_enospc = AsyncMock(return_value=fill) + subject.clear_cached_advert = AsyncMock() + subject.core_uptime = AsyncMock(side_effect=[20, 21]) + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + commands = SimpleNamespace( + remove_contact=AsyncMock(return_value=event("OK")), + add_contact=AsyncMock(return_value=event("OK")), + update_contact=AsyncMock(return_value=event("OK")), + send=AsyncMock(return_value=event("ERROR", {"error_code": 5})), + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_contact_remove_recovery_when_full(set(original)) + + commands.remove_contact.assert_awaited_once_with(target_key) + subject.clear_cached_advert.assert_awaited_once_with(target_key) + commands.add_contact.assert_not_awaited() + commands.update_contact.assert_not_awaited() + self.assertEqual( + subject.set_contact_exact.await_args.args[0]["public_key"], target_key + ) + self.assertEqual(subject.fill_storage_to_enospc.await_count, 1) + self.assertEqual(subject.clear_filler.await_count, 2) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual(subject.contacts.await_count, 6) + step = subject.steps[-1] + self.assertEqual(step["step"], "full_storage_contact_remove_recovery") + self.assertEqual(step["uptime_before_refusal"], 20) + self.assertEqual(step["uptime_after_refusal"], 21) + self.assertTrue(step["remove_acknowledged"]) + self.assertTrue(step["persisted_after_filler_clear"]) + self.assertTrue(step["original_restored"]) + self.assertEqual(step["keyset_sha256"], subject.keyset_digest(original)) + + async def test_full_storage_remove_failure_emergency_restores_contact(self): + subject = self.make_runner() + original = self.full_contacts() + target_key = HIL.make_contact(348)["public_key"] + without_target = { + key_value: dict(value) + for key_value, value in original.items() + if key_value != target_key + } + subject.generated_keys = list(original) + subject.contacts = AsyncMock( + side_effect=[ + original, + HIL.HilFailure("injected after REMOVE ACK"), + without_target, + original, + ] + ) + subject.fill_storage_to_enospc = AsyncMock( + return_value=HIL.ExtraFsFill(131072, 36864, 99, 100) + ) + subject.clear_cached_advert = AsyncMock() + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + commands = SimpleNamespace( + remove_contact=AsyncMock(return_value=event("OK")), + add_contact=AsyncMock(return_value=event("OK")), + update_contact=AsyncMock(return_value=event("OK")), + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "injected after REMOVE ACK"): + await subject.test_contact_remove_recovery_when_full(set(original)) + + commands.remove_contact.assert_awaited_once_with(target_key) + subject.clear_cached_advert.assert_awaited_once_with(target_key) + commands.add_contact.assert_not_awaited() + commands.update_contact.assert_not_awaited() + self.assertEqual( + subject.set_contact_exact.await_args.args[0]["public_key"], target_key + ) + self.assertEqual(subject.clear_filler.await_count, 1) + subject.reboot_and_reconnect.assert_awaited_once() + self.assertEqual( + subject.steps[-1]["step"], + "full_storage_contact_remove_emergency_restore", + ) + self.assertEqual( + subject.steps[-1]["keyset_sha256"], subject.keyset_digest(original) + ) + + async def test_full_storage_remove_error_reconciles_uncertain_mutation(self): + subject = self.make_runner() + original = self.full_contacts() + target_key = HIL.make_contact(348)["public_key"] + subject.generated_keys = list(original) + subject.contacts = AsyncMock(return_value=original) + subject.fill_storage_to_enospc = AsyncMock( + return_value=HIL.ExtraFsFill(131072, 36864, 99, 100) + ) + subject.clear_cached_advert = AsyncMock() + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + commands = SimpleNamespace( + remove_contact=AsyncMock( + return_value=event("ERROR", {"error_code": 5}) + ), + add_contact=AsyncMock(return_value=event("OK")), + update_contact=AsyncMock(return_value=event("OK")), + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "device returned error"): + await subject.test_contact_remove_recovery_when_full(set(original)) + + commands.remove_contact.assert_awaited_once_with(target_key) + subject.clear_cached_advert.assert_awaited_once_with(target_key) + commands.update_contact.assert_not_awaited() + commands.add_contact.assert_not_awaited() + subject.set_contact_exact.assert_awaited_once() + subject.clear_filler.assert_awaited_once() + subject.reboot_and_reconnect.assert_awaited_once() + self.assertEqual(subject.contacts.await_count, 3) + self.assertEqual( + subject.steps[-1]["step"], + "full_storage_contact_remove_emergency_restore", + ) + + async def test_full_storage_remove_fill_failure_still_clears_partial_filler(self): + subject = self.make_runner() + original = self.full_contacts() + target_key = HIL.make_contact(348)["public_key"] + subject.generated_keys = list(original) + subject.contacts = AsyncMock(return_value=original) + subject.fill_storage_to_enospc = AsyncMock( + side_effect=HIL.HilFailure("injected partial fill failure") + ) + subject.clear_cached_advert = AsyncMock() + subject.clear_filler = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + commands = SimpleNamespace(remove_contact=AsyncMock()) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "partial fill failure"): + await subject.test_contact_remove_recovery_when_full(set(original)) + + commands.remove_contact.assert_not_awaited() + subject.clear_cached_advert.assert_awaited_once_with(target_key) + subject.clear_filler.assert_awaited_once() + subject.reboot_and_reconnect.assert_not_awaited() + + async def test_advert_hil_helpers_use_exact_direct_only_commands(self): + subject = self.make_runner() + public_key = HIL.make_contact(12)["public_key"] + token = public_key[:14] + subject.cli = AsyncMock( + side_effect=[ + "HIL contact slot=27", + "HIL advert seeded slot=27", + "HIL advert cleared", + "HIL contact page 13 read failure armed", + "HIL contact page 12 stat failure armed", + ] + ) + + self.assertEqual(await subject.contact_slot(public_key), 27) + self.assertEqual(await subject.seed_cached_advert(public_key), 27) + await subject.clear_cached_advert(public_key) + await subject.arm_contact_read_failure(13) + await subject.arm_contact_stat_failure(12) + + self.assertEqual( + [entry.args[0] for entry in subject.cli.await_args_list], + [ + f"hil extrafs contact-slot {token}", + f"hil extrafs seed-advert {token} CONFIRM", + f"hil extrafs clear-advert {token} CONFIRM", + "hil extrafs fail-read-page 13 CONFIRM", + "hil extrafs fail-stat-page 12 CONFIRM", + ], + ) + + async def test_unread_page_get_contacts_requires_file_io_without_partial_state(self): + subject = self.make_runner() + reader = SimpleNamespace(contact_nb=350, contacts={"stale": {}}) + commands = SimpleNamespace( + send=AsyncMock( + return_value=event("ERROR", {"error_code": 5}) + ) + ) + subject.mc = SimpleNamespace(_reader=reader, commands=commands) + + await subject.expect_contacts_file_io("unit unread page") + + send_call = commands.send.await_args + self.assertEqual(send_call.args[0], b"\x04") + self.assertEqual(send_call.kwargs["timeout"], 20) + self.assertIsNone(reader.contact_nb) + self.assertEqual(reader.contacts, {}) + + async def test_unread_page_get_contacts_rejects_partial_frames_before_error(self): + subject = self.make_runner() + reader = SimpleNamespace(contact_nb=None, contacts={}) + + async def partial_then_error(*_args, **_kwargs): + reader.contact_nb = 350 + reader.contacts = {HIL.make_contact(0)["public_key"]: {}} + return event("ERROR", {"error_code": 5}) + + commands = SimpleNamespace(send=AsyncMock(side_effect=partial_then_error)) + subject.mc = SimpleNamespace(_reader=reader, commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "partial contact stream"): + await subject.expect_contacts_file_io("unit unread page") + + async def test_advert_backed_remove_rolls_back_live_slot_and_exact_inventory(self): + subject = self.make_runner() + baseline = self.full_contacts() + first_key = HIL.make_contact(0)["public_key"] + target_key = HIL.make_contact(349)["public_key"] + without_first = { + key_value: dict(value) + for key_value, value in baseline.items() + if key_value != first_key + } + subject.contacts = AsyncMock( + side_effect=[ + baseline, + without_first, + without_first, + without_first, + without_first, + baseline, + ] + ) + subject.contact_slot = AsyncMock( + side_effect=[10, 300, 300, 300, 300, 10, 300] + ) + subject.seed_cached_advert = AsyncMock(return_value=300) + subject.fill_storage_to_enospc = AsyncMock( + return_value=HIL.ExtraFsFill(131072, 36864, 99, 100) + ) + subject.graceful_reboot_or_enospc_refusal = AsyncMock( + return_value=("refused", 20, 21) + ) + subject.clear_filler = AsyncMock() + subject.clear_cached_advert = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + subject.reboot_and_reconnect = AsyncMock() + commands = SimpleNamespace( + remove_contact=AsyncMock( + side_effect=[ + event("OK"), + event("ERROR", {"error_code": 5}), + ] + ) + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_advert_backed_remove_rollback(set(baseline)) + + self.assertEqual( + [entry.args[0] for entry in commands.remove_contact.await_args_list], + [first_key, target_key], + ) + subject.seed_cached_advert.assert_awaited_once_with(target_key) + subject.clear_cached_advert.assert_awaited_once_with(target_key) + self.assertEqual( + [ + entry.args[0]["public_key"] + for entry in subject.set_contact_exact.await_args_list + ], + [first_key, target_key], + ) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual(subject.contacts.await_count, 6) + step = subject.steps[-1] + self.assertEqual(step["step"], "advert_remove_rollback") + self.assertEqual(step["earlier_slot"], 10) + self.assertEqual(step["target_slot"], 300) + self.assertEqual(step["reboot_outcome"], "refused") + self.assertTrue(step["live_target_and_slot_rolled_back"]) + self.assertEqual( + step["semantic_sha256"], subject.semantic_contact_digest(baseline) + ) + self.assertEqual(step["wire_sha256"], subject.wire_contact_digest(baseline)) + self.assertEqual(step["keyset_sha256"], subject.keyset_digest(baseline)) + + async def test_advert_rollback_unexpected_remove_ok_emergency_restores_slots(self): + subject = self.make_runner() + baseline = self.full_contacts() + first_key = HIL.make_contact(0)["public_key"] + target_key = HIL.make_contact(349)["public_key"] + without_first = { + key_value: dict(value) + for key_value, value in baseline.items() + if key_value != first_key + } + without_both = { + key_value: dict(value) + for key_value, value in without_first.items() + if key_value != target_key + } + subject.contacts = AsyncMock( + side_effect=[baseline, without_first, without_both, baseline] + ) + subject.contact_slot = AsyncMock( + side_effect=[10, 300, 300, 10, 300] + ) + subject.seed_cached_advert = AsyncMock(return_value=300) + subject.fill_storage_to_enospc = AsyncMock( + return_value=HIL.ExtraFsFill(131072, 36864, 99, 100) + ) + subject.graceful_reboot_or_enospc_refusal = AsyncMock() + subject.clear_filler = AsyncMock() + subject.clear_cached_advert = AsyncMock() + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + subject.reboot_and_reconnect = AsyncMock() + commands = SimpleNamespace( + remove_contact=AsyncMock( + side_effect=[event("OK"), event("OK")] + ) + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "expected error code 5"): + await subject.test_advert_backed_remove_rollback(set(baseline)) + + subject.graceful_reboot_or_enospc_refusal.assert_not_awaited() + subject.clear_filler.assert_awaited_once() + subject.clear_cached_advert.assert_awaited_once_with(target_key) + self.assertEqual( + [ + entry.args[0]["public_key"] + for entry in subject.set_contact_exact.await_args_list + ], + [first_key, target_key], + ) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual( + subject.steps[-1]["step"], + "advert_remove_rollback_emergency_restore", + ) + self.assertEqual( + subject.steps[-1]["wire_sha256"], + subject.wire_contact_digest(baseline), + ) + + async def test_read_failure_selector_finds_page_and_live_targets_by_slot(self): + subject = self.make_runner() + subject.contact_slot = AsyncMock(side_effect=[325, 326, 0, 1]) + + failed_key, update_key, remove_key, slots = ( + await subject.select_read_failure_contacts(13) + ) + + self.assertEqual(failed_key, HIL.make_contact(325)["public_key"]) + self.assertEqual(update_key, HIL.make_contact(0)["public_key"]) + self.assertEqual(remove_key, HIL.make_contact(1)["public_key"]) + self.assertEqual( + slots, + { + failed_key: 325, + update_key: 0, + remove_key: 1, + }, + ) + + async def test_contact_read_failure_fails_closed_then_recovers_exactly(self): + subject = self.make_runner() + baseline = self.full_contacts() + failed_key = HIL.make_contact(0)["public_key"] + update_key = HIL.make_contact(25)["public_key"] + remove_key = HIL.make_contact(26)["public_key"] + expected_slots = {failed_key: 0, update_key: 25, remove_key: 26} + status = HIL.ExtraFsStatus(True, 60, 100, 0) + subject.clear_filler = AsyncMock() + subject.status = AsyncMock(side_effect=[status, status, status]) + subject.contacts = AsyncMock(side_effect=[baseline, baseline]) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.select_read_failure_contacts = AsyncMock( + return_value=(failed_key, update_key, remove_key, expected_slots) + ) + subject.arm_contact_read_failure = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.expect_contacts_file_io = AsyncMock() + subject.expect_contact_not_found = AsyncMock() + subject.contact_by_key = AsyncMock( + side_effect=[ + baseline[update_key], + baseline[remove_key], + baseline[update_key], + baseline[remove_key], + baseline[update_key], + ] + ) + subject.set_contact_exact = AsyncMock( + side_effect=[ + event("ERROR", {"error_code": 5}), + event("ERROR", {"error_code": 5}), + ] + ) + subject.contact_slot = AsyncMock(side_effect=[25, 26, 0, 25, 26]) + commands = SimpleNamespace( + remove_contact=AsyncMock( + return_value=event("ERROR", {"error_code": 5}) + ) + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_contact_page_read_failure(0) + + subject.arm_contact_read_failure.assert_awaited_once_with(0) + self.assertEqual(subject.expect_contacts_file_io.await_count, 2) + self.assertEqual(subject.expect_contact_not_found.await_count, 3) + self.assertEqual(subject.set_contact_exact.await_count, 2) + commands.remove_contact.assert_awaited_once_with(remove_key) + self.assertEqual( + [entry.args[0] for entry in subject.contact_slot.await_args_list], + [update_key, remove_key, failed_key, update_key, remove_key], + ) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual(subject.status.await_count, 3) + step = subject.steps[-1] + self.assertEqual(step["step"], "contact_read_failure_recovery") + self.assertEqual(step["page"], 0) + self.assertTrue(step["partial_inventory_rejected"]) + self.assertTrue(step["live_mutations_rolled_back"]) + self.assertTrue(step["live_slots_rolled_back"]) + self.assertEqual(step["baseline_used_kib"], 60) + self.assertEqual(step["final_used_kib"], 60) + self.assertEqual(step["wire_sha256"], subject.wire_contact_digest(baseline)) + self.assertEqual( + step["semantic_sha256"], subject.semantic_contact_digest(baseline) + ) + + async def test_contact_read_failure_rejects_final_storage_usage_change(self): + subject = self.make_runner() + baseline = self.full_contacts() + failed_key = HIL.make_contact(0)["public_key"] + update_key = HIL.make_contact(25)["public_key"] + remove_key = HIL.make_contact(26)["public_key"] + expected_slots = {failed_key: 0, update_key: 25, remove_key: 26} + clean_status = HIL.ExtraFsStatus(True, 60, 100, 0) + leaked_status = HIL.ExtraFsStatus(True, 61, 100, 0) + subject.clear_filler = AsyncMock() + subject.status = AsyncMock( + side_effect=[ + clean_status, + clean_status, + leaked_status, + clean_status, + ] + ) + subject.contacts = AsyncMock( + side_effect=[baseline, baseline, baseline] + ) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.select_read_failure_contacts = AsyncMock( + return_value=(failed_key, update_key, remove_key, expected_slots) + ) + subject.arm_contact_read_failure = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.expect_contacts_file_io = AsyncMock() + subject.expect_contact_not_found = AsyncMock() + subject.contact_by_key = AsyncMock( + side_effect=[ + baseline[update_key], + baseline[remove_key], + baseline[update_key], + baseline[remove_key], + baseline[update_key], + ] + ) + subject.set_contact_exact = AsyncMock( + side_effect=[ + event("ERROR", {"error_code": 5}), + event("ERROR", {"error_code": 5}), + ] + ) + subject.contact_slot = AsyncMock( + side_effect=[25, 26, 0, 25, 26, 0, 25, 26] + ) + commands = SimpleNamespace( + remove_contact=AsyncMock( + return_value=event("ERROR", {"error_code": 5}) + ) + ) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex( + HIL.HilFailure, + "changed ExtraFS usage from 60 KiB to 61 KiB", + ): + await subject.test_contact_page_read_failure(0) + + self.assertEqual(subject.reboot_and_reconnect.await_count, 4) + self.assertEqual(subject.status.await_count, 4) + self.assertEqual( + subject.steps[-1]["step"], + "contact_read_failure_emergency_recovery", + ) + self.assertTrue(subject.steps[-1]["recovered_exactly"]) + + async def test_contact_read_failure_unexpected_add_ok_emergency_recovers(self): + subject = self.make_runner() + baseline = self.full_contacts() + failed_key = HIL.make_contact(0)["public_key"] + update_key = HIL.make_contact(25)["public_key"] + remove_key = HIL.make_contact(26)["public_key"] + expected_slots = {failed_key: 0, update_key: 25, remove_key: 26} + status = HIL.ExtraFsStatus(True, 60, 100, 0) + subject.clear_filler = AsyncMock() + subject.status = AsyncMock(side_effect=[status, status]) + subject.contacts = AsyncMock(side_effect=[baseline, baseline]) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.select_read_failure_contacts = AsyncMock( + return_value=(failed_key, update_key, remove_key, expected_slots) + ) + subject.arm_contact_read_failure = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.expect_contacts_file_io = AsyncMock() + subject.expect_contact_not_found = AsyncMock() + subject.contact_by_key = AsyncMock( + side_effect=[baseline[update_key], baseline[remove_key]] + ) + subject.set_contact_exact = AsyncMock(return_value=event("OK")) + subject.contact_slot = AsyncMock(side_effect=[0, 25, 26]) + commands = SimpleNamespace(remove_contact=AsyncMock()) + subject.mc = SimpleNamespace(commands=commands) + + with self.assertRaisesRegex(HIL.HilFailure, "expected error code 5"): + await subject.test_contact_page_read_failure(0) + + self.assertEqual(subject.reboot_and_reconnect.await_count, 3) + commands.remove_contact.assert_not_awaited() + self.assertEqual( + subject.steps[-1]["step"], + "contact_read_failure_emergency_recovery", + ) + self.assertTrue(subject.steps[-1]["recovered_exactly"]) + self.assertEqual( + subject.steps[-1]["keyset_sha256"], subject.keyset_digest(baseline) + ) + + async def test_contact_stat_failure_blocks_whole_store_then_recovers_exactly(self): + subject = self.make_runner() + baseline = self.full_contacts() + failed_key = HIL.make_contact(0)["public_key"] + update_key = HIL.make_contact(25)["public_key"] + remove_key = HIL.make_contact(26)["public_key"] + expected_slots = {failed_key: 0, update_key: 25, remove_key: 26} + status = HIL.ExtraFsStatus(True, 60, 100, 0) + subject.clear_filler = AsyncMock() + subject.status = AsyncMock(side_effect=[status, status, status]) + subject.contacts = AsyncMock(side_effect=[baseline, baseline]) + subject.contact_page_mask = AsyncMock( + return_value=HIL.EXPECTED_FULL_PAGE_MASK + ) + subject.select_read_failure_contacts = AsyncMock( + return_value=(failed_key, update_key, remove_key, expected_slots) + ) + subject.arm_contact_stat_failure = AsyncMock() + subject.reboot_and_reconnect = AsyncMock() + subject.expect_contacts_file_io = AsyncMock() + subject.set_contact_exact = AsyncMock( + side_effect=[ + event("ERROR", {"error_code": 5}), + event("ERROR", {"error_code": 5}), + ] + ) + subject.contact_slot = AsyncMock(side_effect=[0, 25, 26]) + commands = SimpleNamespace( + remove_contact=AsyncMock( + return_value=event("ERROR", {"error_code": 5}) + ) + ) + subject.mc = SimpleNamespace(commands=commands) + + await subject.test_contact_page_stat_failure(0) + + subject.arm_contact_stat_failure.assert_awaited_once_with(0) + self.assertEqual(subject.expect_contacts_file_io.await_count, 2) + self.assertEqual(subject.set_contact_exact.await_count, 2) + commands.remove_contact.assert_awaited_once_with(remove_key) + self.assertEqual(subject.reboot_and_reconnect.await_count, 2) + self.assertEqual(subject.status.await_count, 3) + self.assertEqual( + [entry.args[0] for entry in subject.contact_slot.await_args_list], + [failed_key, update_key, remove_key], + ) + step = subject.steps[-1] + self.assertEqual(step["step"], "contact_stat_failure_recovery") + self.assertEqual(step["page"], 0) + self.assertTrue(step["whole_source_discovery_rejected"]) + self.assertTrue(step["marker_storage_reclaimed"]) + self.assertEqual(step["baseline_used_kib"], 60) + self.assertEqual(step["final_used_kib"], 60) + self.assertEqual(step["wire_sha256"], subject.wire_contact_digest(baseline)) + self.assertEqual( + step["semantic_sha256"], subject.semantic_contact_digest(baseline) + ) + + +class ReconnectRaceTests(unittest.IsolatedAsyncioTestCase): + def make_runner(self): + config = SimpleNamespace(verbose=False) + identity = SimpleNamespace(device="COM23") + return HIL.StressRunner(config, identity) + + async def test_retries_transient_windows_access_denied(self): + subject = self.make_runner() + subject.connect = AsyncMock(side_effect=[PermissionError(13, "Access denied"), None]) + subject.disconnect = AsyncMock() + + await subject.connect_when_openable(1.0) + + self.assertEqual(subject.connect.await_count, 2) + subject.disconnect.assert_awaited_once() + + async def test_reenumeration_surfaces_typed_reboot_refusal(self): + refusal = event("ERROR", {"error_code": 5}) + error_future = asyncio.get_running_loop().create_future() + error_future.set_result(refusal) + previous = HIL.PortIdentity( + "COM23", + HIL.EXPECTED_USB_SERIAL, + HIL.EXPECTED_USB_VID, + HIL.EXPECTED_USB_PID, + "T1000-E", + "T1000-E", + "MI_00", + "1-8:x.0", + "USB MI_00", + ) + + with self.assertRaises(HIL.RebootRefused) as raised: + await HIL.wait_for_reenumeration( + previous, + 0.1, + port_source=lambda: [], + error_future=error_future, + ) + + self.assertIs(raised.exception.event, refusal) + + async def test_identity_mismatch_is_never_retried(self): + subject = self.make_runner() + subject.connect = AsyncMock( + side_effect=HIL.IdentityGateFailure("wrong node public key") + ) + subject.disconnect = AsyncMock() + + with self.assertRaisesRegex(HIL.IdentityGateFailure, "wrong node"): + await subject.connect_when_openable(1.0) + + subject.connect.assert_awaited_once() + subject.disconnect.assert_awaited_once() + + async def test_graceful_reboot_accepts_verified_file_io_refusal(self): + subject = self.make_runner() + refusal = event("ERROR", {"error_code": 5}) + subject.core_uptime = AsyncMock(side_effect=[20, 21]) + subject.reboot_and_reconnect = AsyncMock( + side_effect=HIL.RebootRefused(refusal) + ) + + outcome = await subject.graceful_reboot_or_enospc_refusal() + + self.assertEqual(outcome, ("refused", 20, 21)) + self.assertEqual(subject.steps[-1]["step"], "advert_rollback_reboot_refused") + + async def test_graceful_reboot_relies_on_inner_reset_proof_after_slow_reconnect(self): + subject = self.make_runner() + subject.core_uptime = AsyncMock(side_effect=[6, 9]) + subject.reboot_and_reconnect = AsyncMock() + + outcome = await subject.graceful_reboot_or_enospc_refusal() + + self.assertEqual(outcome, ("rebooted", 6, 9)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/hil/t1000e_contact_protocol_stress.py b/tools/hil/t1000e_contact_protocol_stress.py new file mode 100644 index 000000000..f4709c91a --- /dev/null +++ b/tools/hil/t1000e_contact_protocol_stress.py @@ -0,0 +1,964 @@ +#!/usr/bin/env python3 +"""Destructive, identity-gated T1000-E USB contact-protocol stress test. + +This focused HIL test assumes the spare T1000-E already contains the 350 +deterministic contacts created by ``t1000e_extrafs_stress.py``. It sends no +LoRa traffic. Every operation is a local USB Companion command. + +The test rejects the requested forbidden-length cases and representative +invalid-type/path encodings for ``CMD_ADD_UPDATE_CONTACT``, checking the +complete contact inventory after each rejection. It then removes and re-adds +one deterministic contact while a paced ``GET_CONTACTS`` response is in +flight. A raw response observer proves the firmware emits a new +``CONTACTS_START`` and that the final completed transaction and the SDK +aggregation are both internally consistent. + +Example (the confirmations are intentionally verbose):: + + python tools/hil/t1000e_contact_protocol_stress.py \ + --port COM23 --allow-destructive \ + --confirm-usb-serial 34A9141999729D5D \ + --expected-node-key-prefix 01234567 \ + --expected-contact-keyset-sha256 \ + 74f58d5b6b29bb7a95655a667768d803b7c12121eee5cab836ebd1da992b8b7c + +Replace ``01234567`` with the current node public-key prefix printed by the +read-only inspection step. It is deliberately supplied at run time because +a full erase regenerates the node identity. + +Progress is written to stderr. The single stdout value is a JSON result. +""" + +from __future__ import annotations + +import argparse +import asyncio +from dataclasses import asdict, dataclass, field +from hashlib import sha256 +import json +from pathlib import Path +import struct +import sys +from types import SimpleNamespace +from typing import Any, Dict, Iterable, List, Optional, Sequence + + +# The sibling harness owns the already-audited USB VID/PID/interface/location +# checks and the strict contact-list retry logic. Loading it rather than +# duplicating those safety gates keeps both destructive tools fail-closed in +# exactly the same way. +try: + import t1000e_extrafs_stress as BASE +except ModuleNotFoundError: # Supports importlib-based unit-test loading. + _HIL_DIR = str(Path(__file__).resolve().parent) + if _HIL_DIR not in sys.path: + sys.path.insert(0, _HIL_DIR) + import t1000e_extrafs_stress as BASE + + +EXPECTED_USB_SERIAL = "34A9141999729D5D" +EXPECTED_CONTACT_COUNT = 350 +MIN_RESTORE_FREE_KIB = 8 +EXPECTED_KEYSET_SHA256 = ( + "74f58d5b6b29bb7a95655a667768d803b7c12121eee5cab836ebd1da992b8b7c" +) +DEFAULT_MUTATION_INDEX = EXPECTED_CONTACT_COUNT - 1 +ERR_CODE_ILLEGAL_ARG = 6 + +CMD_GET_CONTACTS = 4 +CMD_ADD_UPDATE_CONTACT = 9 +CMD_REMOVE_CONTACT = 15 +RESP_CODE_CONTACTS_START = 2 +RESP_CODE_CONTACT = 3 +RESP_CODE_END_OF_CONTACTS = 4 + +CONTACT_UPDATE_MIN_LEN = 136 +CONTACT_UPDATE_GPS_LEN = 144 +CONTACT_UPDATE_LASTMOD_LEN = 148 +FORBIDDEN_CONTACT_UPDATE_LENGTHS = ( + 135, + *range(137, 144), + *range(145, 148), +) +INVALID_CONTACT_TYPES = (0, 5, 255) +# 0x7f and 0xbf encode paths larger than the 64-byte path buffer. 0xc0 +# selects the reserved four-byte hash size. +INVALID_PATH_LENGTHS = (0x7F, 0xBF, 0xC0) + +SEMANTIC_CONTACT_FIELDS = ( + "public_key", + "type", + "flags", + "out_path_hash_mode", + "out_path_len", + "out_path", + "adv_name", + "last_advert", + "adv_lat", + "adv_lon", +) +WIRE_CONTACT_FIELDS = SEMANTIC_CONTACT_FIELDS + ("lastmod",) + + +class ProtocolHilFailure(RuntimeError): + """Raised when an identity, protocol, or restoration invariant fails.""" + + +@dataclass(frozen=True) +class Config: + port: str + allow_destructive: bool + confirm_usb_serial: str + expected_node_key_prefix: str + expected_contact_keyset_sha256: str + mutation_index: int = DEFAULT_MUTATION_INDEX + stream_trigger_contacts: int = 12 + settle_seconds: float = 7.0 + verbose: bool = False + + +@dataclass(frozen=True) +class MalformedCase: + name: str + frame: bytes + + +@dataclass +class StreamSegment: + advertised_count: int + start_frame_index: int + contact_keys: List[str] = field(default_factory=list) + ended: bool = False + end_frame_index: Optional[int] = None + + +class ContactStreamObserver: + """Observe the raw response frames hidden by meshcore's SDK aggregator.""" + + def __init__(self, trigger_contacts: int) -> None: + self.trigger_contacts = trigger_contacts + self.frame_count = 0 + self.segments: List[StreamSegment] = [] + self.ready = asyncio.Event() + self.ended = asyncio.Event() + self.background_failure: Optional[BaseException] = None + + def observe(self, raw: bytes) -> None: + self.frame_count += 1 + if not raw: + return + packet_type = raw[0] + if packet_type == RESP_CODE_CONTACTS_START: + if len(raw) != 5: + raise ProtocolHilFailure( + f"CONTACTS_START has {len(raw)} bytes, expected 5" + ) + if self.segments and self.segments[-1].ended: + raise ProtocolHilFailure( + "unexpected CONTACTS_START after a completed transaction" + ) + self.segments.append( + StreamSegment( + int.from_bytes(raw[1:5], "little"), self.frame_count + ) + ) + return + if packet_type == RESP_CODE_CONTACT: + if len(raw) != 148: + raise ProtocolHilFailure( + f"CONTACT has {len(raw)} bytes, expected 148" + ) + if not self.segments or self.segments[-1].ended: + raise ProtocolHilFailure( + "CONTACT arrived without a valid CONTACTS_START" + ) + current = self.segments[-1] + current.contact_keys.append(raw[1:33].hex()) + if ( + len(self.segments) == 1 + and len(current.contact_keys) >= self.trigger_contacts + ): + self.ready.set() + return + if packet_type == RESP_CODE_END_OF_CONTACTS: + if len(raw) != 5: + raise ProtocolHilFailure( + f"END_OF_CONTACTS has {len(raw)} bytes, expected 5" + ) + if not self.segments or self.segments[-1].ended: + raise ProtocolHilFailure( + "END_OF_CONTACTS arrived without CONTACTS_START" + ) + current = self.segments[-1] + current.ended = True + current.end_frame_index = self.frame_count + self.ended.set() + + def validate_restart( + self, expected_keys: set[str], mutation_after_frame: int + ) -> Dict[str, Any]: + fresh = [ + segment + for segment in self.segments + if segment.start_frame_index > mutation_after_frame + ] + if not fresh: + raise ProtocolHilFailure( + "structural mutation did not produce a fresh CONTACTS_START" + ) + final = self.segments[-1] + if final not in fresh or not final.ended: + raise ProtocolHilFailure( + "the final restarted contact-list transaction did not reach " + "END_OF_CONTACTS" + ) + actual_keys = set(final.contact_keys) + if final.advertised_count != len(expected_keys): + raise ProtocolHilFailure( + f"restarted stream advertised {final.advertised_count}, " + f"expected {len(expected_keys)}" + ) + if len(final.contact_keys) != final.advertised_count: + raise ProtocolHilFailure( + f"restarted stream emitted {len(final.contact_keys)} contacts, " + f"advertised {final.advertised_count}" + ) + if len(actual_keys) != len(final.contact_keys): + raise ProtocolHilFailure("restarted stream contains duplicate keys") + if actual_keys != expected_keys: + raise ProtocolHilFailure("restarted stream has the wrong contact keyset") + return { + "starts": len(self.segments), + "fresh_starts": len(fresh), + "start_counts": [segment.advertised_count for segment in self.segments], + "aborted_contact_counts": [ + len(segment.contact_keys) + for segment in self.segments + if not segment.ended + ], + "completed_count": len(final.contact_keys), + } + + +def _clean(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def require_consent(config: Config) -> None: + if not config.allow_destructive: + raise ProtocolHilFailure("test requires --allow-destructive") + if config.confirm_usb_serial.upper() != EXPECTED_USB_SERIAL: + raise ProtocolHilFailure( + "test requires --confirm-usb-serial " + EXPECTED_USB_SERIAL + ) + node_prefix = config.expected_node_key_prefix.strip().lower() + if not 8 <= len(node_prefix) <= 64: + raise ProtocolHilFailure( + "test requires an 8..64 character --expected-node-key-prefix" + ) + try: + bytes.fromhex(node_prefix) + except ValueError as exc: + raise ProtocolHilFailure( + "--expected-node-key-prefix must be hexadecimal" + ) from exc + if ( + config.expected_contact_keyset_sha256.lower() + != EXPECTED_KEYSET_SHA256 + ): + raise ProtocolHilFailure( + "test requires --expected-contact-keyset-sha256 " + + EXPECTED_KEYSET_SHA256 + ) + if not 0 <= config.mutation_index < EXPECTED_CONTACT_COUNT: + raise ProtocolHilFailure("--mutation-index must be in 0..349") + if not 1 <= config.stream_trigger_contacts <= 100: + raise ProtocolHilFailure( + "--stream-trigger-contacts must be in 1..100" + ) + if config.settle_seconds <= 0: + raise ProtocolHilFailure("--settle-seconds must be greater than zero") + + +def deterministic_keys() -> set[str]: + return { + BASE.make_contact(index)["public_key"] + for index in range(EXPECTED_CONTACT_COUNT) + } + + +def keyset_digest(keys: Iterable[str]) -> str: + return sha256("\n".join(sorted(keys)).encode("ascii")).hexdigest() + + +def _contact_projection( + contact: Dict[str, Any], fields: Sequence[str] +) -> Dict[str, Any]: + return {field: contact.get(field) for field in fields} + + +def semantic_digest(contacts: Dict[str, Dict[str, Any]]) -> str: + canonical = [ + _contact_projection(contacts[key], SEMANTIC_CONTACT_FIELDS) + for key in sorted(contacts) + ] + encoded = json.dumps( + canonical, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def wire_snapshot_digest(contacts: Dict[str, Dict[str, Any]]) -> str: + """Digest every contact field emitted on GET_CONTACTS, including lastmod.""" + + canonical = [ + _contact_projection(contacts[key], WIRE_CONTACT_FIELDS) + for key in sorted(contacts) + ] + encoded = json.dumps( + canonical, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return sha256(encoded).hexdigest() + + +def expected_semantic_digest() -> str: + contacts = { + contact["public_key"]: contact + for contact in ( + BASE.make_contact(index) for index in range(EXPECTED_CONTACT_COUNT) + ) + } + return semantic_digest(contacts) + + +def validate_exact_snapshot( + contacts: Dict[str, Dict[str, Any]], *, require_canonical_fields: bool +) -> Dict[str, str | int]: + expected = deterministic_keys() + actual = set(contacts) + digest = keyset_digest(actual) + if len(contacts) != EXPECTED_CONTACT_COUNT: + raise ProtocolHilFailure( + f"contact inventory has {len(contacts)}, expected 350" + ) + if actual != expected or digest != EXPECTED_KEYSET_SHA256: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise ProtocolHilFailure( + "contact inventory is not the deterministic 350-key table; " + f"digest={digest}, missing={[key[:12] for key in missing[:4]]}, " + f"extra={[key[:12] for key in extra[:4]]}" + ) + content = semantic_digest(contacts) + if require_canonical_fields and content != expected_semantic_digest(): + raise ProtocolHilFailure( + "deterministic keys have unexpected contact fields; " + f"semantic digest={content}" + ) + return { + "count": len(contacts), + "keyset_sha256": digest, + "semantic_sha256": content, + "wire_sha256": wire_snapshot_digest(contacts), + } + + +def encode_contact_update( + contact: Dict[str, Any], *, include_lastmod: bool = True +) -> bytes: + """Encode the firmware's fixed-width local contact update structure.""" + + public_key = bytes.fromhex(contact["public_key"]) + if len(public_key) != 32: + raise ProtocolHilFailure("contact public key must be exactly 32 bytes") + path_len = int(contact["out_path_len"]) + if path_len < 0: + encoded_path_len = 0xFF + else: + mode = int(contact["out_path_hash_mode"]) + encoded_path_len = (path_len & 0x3F) | ((mode & 0x03) << 6) + path = bytes.fromhex(_clean(contact.get("out_path"))) + if len(path) > 64: + raise ProtocolHilFailure("contact path exceeds 64 bytes") + name = _clean(contact.get("adv_name")).encode("utf-8")[:32] + frame = bytearray((CMD_ADD_UPDATE_CONTACT,)) + frame.extend(public_key) + frame.extend( + ( + int(contact["type"]) & 0xFF, + int(contact["flags"]) & 0xFF, + encoded_path_len, + ) + ) + frame.extend(path.ljust(64, b"\0")) + frame.extend(name.ljust(32, b"\0")) + frame.extend(struct.pack(" List[MalformedCase]: + canonical = encode_contact_update(contact) + cases = [ + MalformedCase(f"forbidden_length_{length}", canonical[:length]) + for length in FORBIDDEN_CONTACT_UPDATE_LENGTHS + ] + for invalid_type in INVALID_CONTACT_TYPES: + frame = bytearray(canonical) + frame[33] = invalid_type + cases.append(MalformedCase(f"invalid_type_{invalid_type}", bytes(frame))) + for invalid_path in INVALID_PATH_LENGTHS: + frame = bytearray(canonical) + frame[35] = invalid_path + cases.append(MalformedCase(f"invalid_path_{invalid_path:02x}", bytes(frame))) + return cases + + +def _event_name(event: Any) -> str: + return _clean(getattr(getattr(event, "type", None), "name", "")) + + +def require_event(event: Any, expected: str, operation: str) -> Any: + name = _event_name(event) + if name != expected: + raise ProtocolHilFailure( + f"{operation} returned {name or ''}: " + f"{getattr(event, 'payload', None)!r}; expected {expected}" + ) + return event + + +def require_illegal_arg(event: Any, operation: str) -> None: + require_event(event, "ERROR", operation) + payload = getattr(event, "payload", {}) + if payload.get("error_code") != ERR_CODE_ILLEGAL_ARG: + raise ProtocolHilFailure( + f"{operation} returned error {payload!r}; expected " + "ERR_CODE_ILLEGAL_ARG (6)" + ) + + +class ProtocolRunner: + def __init__(self, config: Config, identity: Any) -> None: + self.config = config + self.identity = identity + # StressRunner only needs this subset for connect(), contacts(), and + # manual-contact-mode verification. + base_config = SimpleNamespace( + expected_node_key_prefix=config.expected_node_key_prefix, + contact_count=EXPECTED_CONTACT_COUNT, + channel_index=39, + contact_enumeration_passes=None, + expected_contact_keyset_sha256=EXPECTED_KEYSET_SHA256, + settle_seconds=config.settle_seconds, + reenum_timeout=35.0, + verbose=config.verbose, + ) + self.base = BASE.StressRunner(base_config, identity) + self.steps: List[Dict[str, Any]] = [] + self.original_manual_add_contacts: Optional[bool] = None + self.original_autoadd_config: Optional[int] = None + self.original_target_contact: Optional[Dict[str, Any]] = None + self.baseline_wire_sha256: Optional[str] = None + self.restoration_armed = False + + def progress(self, message: str) -> None: + if self.config.verbose: + print(message, file=sys.stderr, flush=True) + + def record(self, step: str, **details: Any) -> None: + self.steps.append({"step": step, **details}) + self.progress(step) + + async def snapshot(self, *, canonical: bool = True) -> Dict[str, Dict[str, Any]]: + contacts = await self.base.contacts() + details = validate_exact_snapshot( + contacts, require_canonical_fields=canonical + ) + self.record("contact_snapshot", **details) + return contacts + + async def freeze_contact_admission_for_test(self) -> None: + event = self.base._require_event( + await self.base.mc.commands.send_appstart(), "SELF_INFO" + ) + autoadd = self.base._require_event( + await self.base.mc.commands.get_autoadd_config(), "AUTOADD_CONFIG" + ) + self.original_manual_add_contacts = bool( + event.payload.get("manual_add_contacts") + ) + self.original_autoadd_config = int(autoadd.payload.get("config", 0)) + if not self.original_manual_add_contacts: + await self.base.set_manual_contact_mode(True) + if self.original_autoadd_config != 0: + await self.base.set_autoadd_config(0) + self.record( + "contact_admission_frozen", + original_manual_add_contacts=self.original_manual_add_contacts, + original_autoadd_config=self.original_autoadd_config, + ) + + async def test_malformed_updates( + self, baseline: Dict[str, Dict[str, Any]] + ) -> None: + try: + from meshcore.events import EventType + except ImportError as exc: + raise ProtocolHilFailure("meshcore Python package is unavailable") from exc + baseline_semantic = semantic_digest(baseline) + baseline_wire = wire_snapshot_digest(baseline) + target = BASE.make_contact(self.config.mutation_index) + completed: List[str] = [] + for case in malformed_cases(target): + if not case.frame or case.frame[0] != CMD_ADD_UPDATE_CONTACT: + raise AssertionError("malformed test escaped the local contact command") + result = await self.base.mc.commands.send( + case.frame, [EventType.ERROR], timeout=8.0 + ) + require_illegal_arg(result, case.name) + after = await self.snapshot(canonical=True) + if semantic_digest(after) != baseline_semantic: + raise ProtocolHilFailure( + f"{case.name} changed contact content despite rejection" + ) + if wire_snapshot_digest(after) != baseline_wire: + raise ProtocolHilFailure( + f"{case.name} changed contact lastmod despite rejection" + ) + completed.append(case.name) + self.record( + "malformed_contact_updates", + cases=len(completed), + names=completed, + expected_error="ERR_CODE_ILLEGAL_ARG", + unchanged_after_each=True, + ) + + async def test_structural_restart(self) -> None: + expected_keys = deterministic_keys() + if self.original_target_contact is None: + raise ProtocolHilFailure("baseline target contact was not captured") + target = dict(self.original_target_contact) + target_key = target["public_key"] + reader = getattr(self.base.mc, "_reader", None) + if reader is None or not hasattr(reader, "handle_rx"): + raise ProtocolHilFailure("meshcore reader cannot expose raw responses") + + observer = ContactStreamObserver(self.config.stream_trigger_contacts) + original_handle_rx = reader.handle_rx + + async def observed_handle_rx(data: bytearray) -> None: + try: + observer.observe(bytes(data)) + except BaseException as exc: + # SerialConnection dispatches reader calls as background tasks. + # Retain observer failures for the foreground test while still + # feeding the SDK parser so cleanup can drain the transaction. + if observer.background_failure is None: + observer.background_failure = exc + observer.ready.set() + observer.ended.set() + await original_handle_rx(data) + + reader.handle_rx = observed_handle_rx + stream_task: Optional[asyncio.Task[Any]] = None + mutation_after_frame = -1 + try: + stream_task = asyncio.create_task( + self.base.mc.commands.get_contacts(lastmod=0, timeout=20) + ) + await asyncio.wait_for(observer.ready.wait(), timeout=5.0) + if observer.background_failure is not None: + raise ProtocolHilFailure( + f"raw contact observer failed: {observer.background_failure}" + ) + if not observer.segments or observer.segments[0].ended: + raise ProtocolHilFailure( + "initial contact stream ended before structural mutation" + ) + if observer.segments[0].advertised_count != EXPECTED_CONTACT_COUNT: + raise ProtocolHilFailure( + "initial contact stream advertised " + f"{observer.segments[0].advertised_count}, expected 350" + ) + mutation_after_frame = observer.frame_count + + removed = await self.base.mc.commands.remove_contact(target_key) + require_event(removed, "OK", "remove during contact stream") + try: + from meshcore.events import EventType + except ImportError as exc: + raise ProtocolHilFailure( + "meshcore Python package is unavailable" + ) from exc + # Use the supported 148-byte form so the original lastmod survives + # the structural test and incremental sync clients see no phantom + # update after the table has been restored. + readded = await self.base.mc.commands.send( + encode_contact_update(target), + [EventType.OK, EventType.ERROR], + timeout=8.0, + ) + require_event(readded, "OK", "re-add during contact stream") + + stream_event = await asyncio.wait_for(stream_task, timeout=20.0) + if observer.background_failure is not None: + raise ProtocolHilFailure( + f"raw contact observer failed: {observer.background_failure}" + ) + require_event(stream_event, "CONTACTS", "paced contact stream") + sdk_contacts = getattr(stream_event, "payload", None) + if not isinstance(sdk_contacts, dict): + raise ProtocolHilFailure("SDK CONTACTS payload is not a dictionary") + sdk_details = validate_exact_snapshot( + dict(sdk_contacts), require_canonical_fields=True + ) + reader_count = getattr(reader, "contact_nb", None) + if reader_count != EXPECTED_CONTACT_COUNT: + raise ProtocolHilFailure( + f"SDK retained CONTACTS_START count {reader_count!r}, expected 350" + ) + raw_details = observer.validate_restart( + expected_keys, mutation_after_frame + ) + self.record( + "structural_stream_restart", + mutation_index=self.config.mutation_index, + mutation_after_frame=mutation_after_frame, + raw=raw_details, + sdk=sdk_details, + target_restored=True, + ) + finally: + # If a mutation assertion fails, let the in-device iterator finish + # before the outer restoration sends another GET_CONTACTS command. + if observer.segments and not observer.segments[-1].ended: + try: + await asyncio.wait_for(observer.ended.wait(), timeout=5.0) + except asyncio.TimeoutError: + pass + if stream_task is not None and not stream_task.done(): + try: + await asyncio.wait_for(asyncio.shield(stream_task), timeout=5.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + stream_task.cancel() + try: + await stream_task + except asyncio.CancelledError: + pass + elif stream_task is not None: + # Retrieve a terminal exception when the foreground failed + # before it awaited the stream task. + try: + stream_task.result() + except (asyncio.CancelledError, Exception): + pass + reader.handle_rx = original_handle_rx + + async def restore(self) -> None: + """Restore the one permitted target and the admission-mode setting.""" + + errors: List[str] = [] + if self.restoration_armed and self.base.mc is None: + errors.append("contact restoration failed: device is disconnected") + elif self.restoration_armed and self.original_target_contact is None: + errors.append("contact restoration failed: baseline target is missing") + elif self.restoration_armed: + target = dict(self.original_target_contact) + try: + current = await self.base.contacts() + unexpected = set(current) - deterministic_keys() + if unexpected: + raise ProtocolHilFailure( + "refusing automatic cleanup with unexpected keys: " + + ", ".join(key[:12] for key in sorted(unexpected)[:4]) + ) + try: + from meshcore.events import EventType + except ImportError as exc: + raise ProtocolHilFailure( + "meshcore Python package is unavailable" + ) from exc + event = await self.base.mc.commands.send( + encode_contact_update(target), + [EventType.OK, EventType.ERROR], + timeout=8.0, + ) + require_event(event, "OK", "restore deterministic contact") + await asyncio.sleep(self.config.settle_seconds) + # A live-RAM GET_CONTACTS is not persistence proof. The HIL + # preflight guarantees headroom and a graceful reboot forces + # all lazy page writes to commit or explicitly refuses reset. + await self.base.reboot_and_reconnect() + restored = await self.base.contacts() + details = validate_exact_snapshot( + restored, require_canonical_fields=True + ) + if ( + self.baseline_wire_sha256 is not None + and wire_snapshot_digest(restored) + != self.baseline_wire_sha256 + ): + raise ProtocolHilFailure( + "post-reboot contact table does not match the exact " + "baseline wire snapshot" + ) + self.record( + "contact_table_restored", + persistence="verified_after_graceful_reboot", + **details, + ) + except BaseException as exc: + errors.append(f"contact restoration failed: {exc}") + + if self.original_autoadd_config is not None and self.base.mc is None: + errors.append( + "autoadd config restoration failed: device is disconnected" + ) + elif self.original_autoadd_config is not None: + try: + current = self.base._require_event( + await self.base.mc.commands.get_autoadd_config(), + "AUTOADD_CONFIG", + ) + current_config = int(current.payload.get("config", -1)) + if current_config != self.original_autoadd_config: + await self.base.set_autoadd_config( + self.original_autoadd_config + ) + self.record( + "autoadd_config_restored", + autoadd_config=self.original_autoadd_config, + ) + except BaseException as exc: + errors.append(f"autoadd config restoration failed: {exc}") + + if self.original_manual_add_contacts is not None and self.base.mc is None: + errors.append( + "contact admission restoration failed: device is disconnected" + ) + elif self.original_manual_add_contacts is not None: + try: + current = self.base._require_event( + await self.base.mc.commands.send_appstart(), "SELF_INFO" + ) + current_mode = bool(current.payload.get("manual_add_contacts")) + if current_mode != self.original_manual_add_contacts: + await self.base.set_manual_contact_mode( + self.original_manual_add_contacts + ) + self.record( + "contact_admission_restored", + manual_add_contacts=self.original_manual_add_contacts, + ) + except BaseException as exc: + errors.append(f"contact admission restoration failed: {exc}") + if errors: + raise ProtocolHilFailure("; ".join(errors)) + + async def execute(self) -> None: + await self.base.connect() + connected = next( + ( + step + for step in reversed(self.base.steps) + if step.get("step") == "connected" + ), + {}, + ) + self.record( + "connected", + port=connected.get("port", self.identity.device), + usb_serial=connected.get("usb_serial", self.identity.serial_number), + node_key_prefix=connected.get("node_key_prefix"), + device=connected.get("device"), + ) + await self.base.clear_filler() + storage = await self.base.status() + free_kib = storage.total_kib - storage.used_kib + if free_kib < MIN_RESTORE_FREE_KIB: + raise ProtocolHilFailure( + f"only {free_kib} KiB is free after clearing the HIL filler; " + f"at least {MIN_RESTORE_FREE_KIB} KiB is required for durable " + "contact restoration" + ) + self.record( + "persistence_headroom", + used_kib=storage.used_kib, + total_kib=storage.total_kib, + free_kib=free_kib, + hil_filler_cleared=True, + ) + baseline = await self.snapshot(canonical=True) + target_key = BASE.make_contact(self.config.mutation_index)["public_key"] + self.original_target_contact = dict(baseline[target_key]) + self.baseline_wire_sha256 = wire_snapshot_digest(baseline) + self.restoration_armed = True + await self.freeze_contact_admission_for_test() + await self.test_malformed_updates(baseline) + await self.test_structural_restart() + + +async def run(config: Config) -> Dict[str, Any]: + require_consent(config) + identity = BASE.resolve_initial_port(config.port) + runner = ProtocolRunner(config, identity) + result: Dict[str, Any] = { + "schema_version": 1, + "tool": "t1000e_contact_protocol_stress", + "ok": False, + "identity": asdict(identity), + "steps": runner.steps, + "failure": None, + } + primary_error: Optional[BaseException] = None + try: + await runner.execute() + except BaseException as exc: + primary_error = exc + try: + await runner.restore() + except BaseException as exc: + if primary_error is None: + primary_error = exc + else: + primary_error = ProtocolHilFailure( + f"test failed ({primary_error}); cleanup also failed ({exc})" + ) + try: + await runner.base.disconnect() + except BaseException as exc: + if primary_error is None: + primary_error = exc + else: + primary_error = ProtocolHilFailure( + f"test failed ({primary_error}); disconnect also failed ({exc})" + ) + if primary_error is None: + result["ok"] = True + else: + result["failure"] = { + "type": type(primary_error).__name__, + "message": str(primary_error), + } + return result + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", help="T1000-E USB Companion port") + parser.add_argument("--allow-destructive", action="store_true") + parser.add_argument("--confirm-usb-serial", default="") + parser.add_argument("--expected-node-key-prefix", default="") + parser.add_argument("--expected-contact-keyset-sha256", default="") + parser.add_argument("--mutation-index", type=int, default=DEFAULT_MUTATION_INDEX) + parser.add_argument("--stream-trigger-contacts", type=int, default=12) + parser.add_argument("--settle-seconds", type=float, default=7.0) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--pretty", action="store_true") + parser.add_argument( + "--self-test", + action="store_true", + help="run pure offline invariants without importing serial hardware", + ) + return parser + + +def config_from_args(args: argparse.Namespace) -> Config: + if not args.port: + raise ProtocolHilFailure("--port is required unless --self-test is used") + config = Config( + port=args.port, + allow_destructive=args.allow_destructive, + confirm_usb_serial=args.confirm_usb_serial, + expected_node_key_prefix=args.expected_node_key_prefix, + expected_contact_keyset_sha256=args.expected_contact_keyset_sha256, + mutation_index=args.mutation_index, + stream_trigger_contacts=args.stream_trigger_contacts, + settle_seconds=args.settle_seconds, + verbose=args.verbose, + ) + require_consent(config) + return config + + +def offline_self_test() -> Dict[str, Any]: + expected = { + BASE.make_contact(index)["public_key"]: BASE.make_contact(index) + for index in range(EXPECTED_CONTACT_COUNT) + } + snapshot = validate_exact_snapshot( + expected, require_canonical_fields=True + ) + cases = malformed_cases(BASE.make_contact(DEFAULT_MUTATION_INDEX)) + lengths = [ + len(case.frame) + for case in cases + if case.name.startswith("forbidden_length_") + ] + assert tuple(lengths) == FORBIDDEN_CONTACT_UPDATE_LENGTHS + assert all(case.frame[0] == CMD_ADD_UPDATE_CONTACT for case in cases) + + observer = ContactStreamObserver(trigger_contacts=2) + keys = sorted(deterministic_keys()) + observer.observe(bytes((RESP_CODE_CONTACTS_START,)) + struct.pack(" int: + parser = build_argument_parser() + args = parser.parse_args(argv) + if args.self_test: + result = offline_self_test() + else: + try: + result = asyncio.run(run(config_from_args(args))) + except BaseException as exc: + result = { + "schema_version": 1, + "tool": "t1000e_contact_protocol_stress", + "ok": False, + "steps": [], + "failure": {"type": type(exc).__name__, "message": str(exc)}, + } + print(json.dumps(result, indent=2 if args.pretty else None, sort_keys=True)) + return 0 if result.get("ok") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/hil/t1000e_extrafs_stress.py b/tools/hil/t1000e_extrafs_stress.py new file mode 100644 index 000000000..8cf34f131 --- /dev/null +++ b/tools/hil/t1000e_extrafs_stress.py @@ -0,0 +1,3878 @@ +#!/usr/bin/env python3 +"""Identity-gated T1000-E ExtraFS hardware stress harness. + +The default ``inspect`` scenario is read-only. The ``comprehensive`` scenario +fills the contact table, fills internal ExtraFS, checks channel and lazy +contact add/update/remove behavior at ENOSPC, reboots, and verifies persistence. +Optional page +payload/occupancy corruption, awake-USB unsafe-reset, and cleanup phases are +separate flags. A +serial run cannot prove event-driven sleep while USB is attached; use the BLE +sleep HIL procedure for that distinct test. + +This tool never transmits a LoRa message or advertisement. It talks only to +the USB Companion interface and compile-time ``MESHCORE_EXTRAFS_HIL`` commands. + +Example destructive invocation for the dedicated spare board:: + + python tools/hil/t1000e_extrafs_stress.py \ + --port COM23 --scenario comprehensive \ + --allow-destructive --confirm-usb-serial 34A9141999729D5D + +Read-only repeated enumeration of a stable contact table:: + + python tools/hil/t1000e_extrafs_stress.py \ + --port COM23 --scenario inspect --contact-enumeration-passes 25 + +All progress is written to stderr. A single JSON result is written to stdout. +""" + +from __future__ import annotations + +import argparse +import asyncio +from dataclasses import asdict, dataclass +from hashlib import sha256 +import json +import re +import sys +import time +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence + + +EXPECTED_USB_SERIAL = "34A9141999729D5D" +KNOWN_PRE_ERASE_NODE_KEY_PREFIX = "f9ad7082" +EXPECTED_USB_VID = 0x239A +EXPECTED_USB_PID = 0x8029 +EXPECTED_USB_INTERFACE = "MI_00" +EXPECTED_MODEL_TOKEN = "T1000-E" +EXPECTED_STORAGE_KIB = 100 +EXPECTED_CONTACTS_PER_PAGE = 25 +EXPECTED_FULL_PAGE_MASK = (1 << EXPECTED_CONTACTS_PER_PAGE) - 1 +# The destructive filler deliberately closes and read-verifies every 1 KiB +# extent. Near ENOSPC that can require substantially longer than the normal +# command timeout even though the device and filesystem remain healthy. +EXTRAFS_FILL_TIMEOUT_SECONDS = 45.0 +CONTACT_PAGE_REPAIR_TIMEOUT_SECONDS = 10.0 +ENOSPC_REBOOT_REFUSAL_TIMEOUT_SECONDS = 30.0 + +HIL_STATUS_RE = re.compile( + r"HIL ExtraFS active=(?P[01]) " + r"used=(?P\d+)KiB total=(?P\d+)KiB " + r"fill=(?P\d+)" +) +HIL_FILL_RE = re.compile( + r"HIL ExtraFS fill requested=(?P\d+) " + r"written=(?P\d+) used=(?P\d+)KiB " + r"total=(?P\d+)KiB" +) +HIL_PAGE_MASK_RE = re.compile( + r"HIL contact page (?P\d+) occupied=(?P[0-9A-Fa-f]{8})" +) +HIL_CONTACT_SLOT_RE = re.compile(r"HIL contact slot=(?P\d+)") +HIL_ADVERT_SEED_RE = re.compile(r"HIL advert seeded slot=(?P\d+)") + + +class HilFailure(RuntimeError): + """Raised when a safety gate or hardware invariant fails.""" + + +class IdentityGateFailure(HilFailure): + """Raised when an opened device is not the explicitly selected spare.""" + + +class RebootRefused(HilFailure): + """Raised when firmware answers a requested reboot with an error.""" + + def __init__(self, event: Any) -> None: + self.event = event + super().__init__( + "reboot was refused by firmware: " + f"{getattr(event, 'payload', None)!r}" + ) + + +@dataclass(frozen=True) +class PortIdentity: + device: str + serial_number: str + vid: int + pid: int + description: str + product: str + interface: str + location: str + hwid: str + + +@dataclass(frozen=True) +class ExtraFsStatus: + active: bool + used_kib: int + total_kib: int + filler_bytes: int + + +@dataclass(frozen=True) +class ExtraFsFill: + requested_bytes: int + written_bytes: int + used_kib: int + total_kib: int + + +@dataclass(frozen=True) +class ContactPageMask: + page: int + occupied_mask: int + + +@dataclass(frozen=True) +class ReenumerationResult: + identity: PortIdentity + observed_absence: bool + + +@dataclass(frozen=True) +class RunConfig: + port: str + scenario: str + allow_destructive: bool + confirm_usb_serial: Optional[str] + expected_node_key_prefix: Optional[str] + contact_count: int + fill_bytes: int + channel_index: int + settle_seconds: float + reenum_timeout: float + allow_existing_contacts: bool + cleanup: bool + corrupt_page: Optional[int] + unsafe_reset: bool + contact_enumeration_passes: Optional[int] + expected_contact_keyset_sha256: Optional[str] + verbose: bool + # Keep opt-ins at the end with safe defaults so older positional callers + # retain their original field mapping. + corrupt_occupied_page: Optional[int] = None + advert_remove_rollback: bool = False + fail_read_page: Optional[int] = None + fail_stat_page: Optional[int] = None + + +def _clean(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def _upper(value: Any) -> str: + return _clean(value).upper() + + +def port_identity(info: Any) -> PortIdentity: + """Copy the useful pyserial ListPortInfo fields into stable plain data.""" + + return PortIdentity( + device=_clean(getattr(info, "device", "")), + serial_number=_upper(getattr(info, "serial_number", "")), + vid=int(getattr(info, "vid", 0) or 0), + pid=int(getattr(info, "pid", 0) or 0), + description=_clean(getattr(info, "description", "")), + product=_clean(getattr(info, "product", "")), + interface=_clean(getattr(info, "interface", "")), + location=_clean(getattr(info, "location", "")), + hwid=_clean(getattr(info, "hwid", "")), + ) + + +def validate_runtime_identity( + identity: PortIdentity, + *, + requested_port: Optional[str] = None, + expected_location: Optional[str] = None, +) -> None: + """Reject anything other than the locked T1000-E Companion interface.""" + + if requested_port and _upper(identity.device) != _upper(requested_port): + raise HilFailure( + f"resolved port {identity.device!r} does not match requested " + f"port {requested_port!r}" + ) + if identity.serial_number != EXPECTED_USB_SERIAL: + raise HilFailure( + f"USB serial is {identity.serial_number!r}, expected the locked " + f"spare {EXPECTED_USB_SERIAL}" + ) + if (identity.vid, identity.pid) != (EXPECTED_USB_VID, EXPECTED_USB_PID): + raise HilFailure( + f"USB VID:PID is {identity.vid:04X}:{identity.pid:04X}, expected " + f"{EXPECTED_USB_VID:04X}:{EXPECTED_USB_PID:04X}" + ) + interface_text = f"{identity.interface} {identity.hwid}".upper() + # pyserial's Windows backend parses MI_00 but exposes it only as the + # ``:x.0`` suffix of ``location``; unlike Linux it does not populate the + # ``interface`` property or retain MI_00 in ``hwid``. + is_companion_interface = ( + EXPECTED_USB_INTERFACE in interface_text + or identity.location.lower().endswith(":x.0") + ) + if not is_companion_interface: + raise HilFailure( + f"{identity.device} is not the locked {EXPECTED_USB_INTERFACE} " + "Companion interface" + ) + model_text = ( + f"{identity.description} {identity.product} {identity.hwid}".upper() + ) + if "T1000" not in model_text and "8029" not in model_text: + raise HilFailure( + f"{identity.device} does not identify as a T1000-E runtime port" + ) + if expected_location and identity.location != expected_location: + raise HilFailure( + f"USB location changed from {expected_location!r} to " + f"{identity.location!r}; refusing to follow a different device" + ) + + +def parse_extrafs_status(text: str) -> ExtraFsStatus: + match = HIL_STATUS_RE.fullmatch(text.strip()) + if not match: + raise HilFailure(f"unexpected ExtraFS status reply: {text!r}") + values = {name: int(value) for name, value in match.groupdict().items()} + if values["used"] > values["total"]: + raise HilFailure("ExtraFS reports used space greater than total space") + return ExtraFsStatus( + active=bool(values["active"]), + used_kib=values["used"], + total_kib=values["total"], + filler_bytes=values["fill"], + ) + + +def parse_extrafs_fill(text: str) -> ExtraFsFill: + match = HIL_FILL_RE.fullmatch(text.strip()) + if not match: + raise HilFailure(f"unexpected ExtraFS fill reply: {text!r}") + values = {name: int(value) for name, value in match.groupdict().items()} + if values["written"] > values["requested"]: + raise HilFailure("ExtraFS filler wrote more bytes than requested") + if values["used"] > values["total"]: + raise HilFailure("ExtraFS reports used space greater than total space") + return ExtraFsFill( + requested_bytes=values["requested"], + written_bytes=values["written"], + used_kib=values["used"], + total_kib=values["total"], + ) + + +def parse_contact_page_mask(text: str) -> ContactPageMask: + match = HIL_PAGE_MASK_RE.fullmatch(text.strip()) + if not match: + raise HilFailure(f"unexpected contact-page mask reply: {text!r}") + page = int(match.group("page"), 10) + if not 0 <= page <= 13: + raise HilFailure(f"contact-page mask reported invalid page {page}") + return ContactPageMask(page, int(match.group("mask"), 16)) + + +def parse_contact_slot(text: str, *, operation: str = "contact slot") -> int: + match = HIL_CONTACT_SLOT_RE.fullmatch(text.strip()) + if not match: + raise HilFailure(f"unexpected {operation} reply: {text!r}") + slot = int(match.group("slot"), 10) + if not 0 <= slot < 350: + raise HilFailure(f"{operation} reported invalid slot {slot}") + return slot + + +def parse_advert_seed_slot(text: str) -> int: + match = HIL_ADVERT_SEED_RE.fullmatch(text.strip()) + if not match: + raise HilFailure(f"unexpected advert seed reply: {text!r}") + slot = int(match.group("slot"), 10) + if not 0 <= slot < 350: + raise HilFailure(f"advert seed reported invalid slot {slot}") + return slot + + +def make_contact(index: int) -> Dict[str, Any]: + """Return a deterministic, non-radio synthetic Companion contact.""" + + if index < 0: + raise ValueError("contact index must not be negative") + public_key = sha256(f"meshcore-t1000e-extrafs-hil:{index}".encode()).hexdigest() + return { + "public_key": public_key, + "type": 1, + "flags": 0, + "out_path_hash_mode": -1, + "out_path_len": -1, + "out_path": "", + "adv_name": f"HIL-{index:03d}", + "last_advert": 1_700_000_000 + index, + "adv_lat": 0.0, + "adv_lon": 0.0, + } + + +def encode_exact_contact_frame(contact: Dict[str, Any]) -> bytes: + """Encode the 148-byte contact form, including its original lastmod.""" + + try: + public_key = bytes.fromhex(str(contact["public_key"])) + contact_type = int(contact["type"]) + flags = int(contact["flags"]) + path_len = int(contact["out_path_len"]) + path_hash_mode = int(contact["out_path_hash_mode"]) + path = bytes.fromhex(str(contact.get("out_path", ""))) + name = str(contact["adv_name"]).encode("utf-8") + last_advert = int(contact["last_advert"]) + latitude = round(float(contact["adv_lat"]) * 1_000_000) + longitude = round(float(contact["adv_lon"]) * 1_000_000) + lastmod = int(contact["lastmod"]) + except (KeyError, TypeError, ValueError) as exc: + raise HilFailure(f"cannot encode exact contact: {exc}") from exc + + if len(public_key) != 32: + raise HilFailure("exact contact public key is not 32 bytes") + if not 0 <= contact_type <= 0xFF or not 0 <= flags <= 0xFF: + raise HilFailure("exact contact type or flags is outside one byte") + if path_len == -1: + encoded_path_len = 0xFF + expected_path_bytes = 0 + elif 0 <= path_len <= 0x3F and 0 <= path_hash_mode <= 3: + encoded_path_len = path_len | (path_hash_mode << 6) + expected_path_bytes = path_len * (path_hash_mode + 1) + else: + raise HilFailure("exact contact path metadata is invalid") + if len(path) != expected_path_bytes or len(path) > 64: + raise HilFailure( + f"exact contact path is {len(path)} bytes, expected " + f"{expected_path_bytes}" + ) + if len(name) > 31 or b"\0" in name: + raise HilFailure("exact contact name is not a NUL-free 31-byte string") + if not 0 <= last_advert <= 0xFFFFFFFF or not 0 <= lastmod <= 0xFFFFFFFF: + raise HilFailure("exact contact timestamp is outside uint32") + if ( + not -(1 << 31) <= latitude < (1 << 31) + or not -(1 << 31) <= longitude < (1 << 31) + ): + raise HilFailure("exact contact coordinates are outside int32") + + frame = ( + b"\x09" + + public_key + + bytes((contact_type, flags, encoded_path_len)) + + path.ljust(64, b"\0") + + name.ljust(32, b"\0") + + last_advert.to_bytes(4, "little") + + latitude.to_bytes(4, "little", signed=True) + + longitude.to_bytes(4, "little", signed=True) + + lastmod.to_bytes(4, "little") + ) + if len(frame) != 148: + raise HilFailure(f"exact contact frame is {len(frame)} bytes, expected 148") + return frame + + +def require_destructive_consent(config: RunConfig) -> None: + destructive = ( + config.scenario != "inspect" + or config.cleanup + or config.corrupt_page is not None + or getattr(config, "corrupt_occupied_page", None) is not None + or getattr(config, "advert_remove_rollback", False) + or getattr(config, "fail_read_page", None) is not None + or getattr(config, "fail_stat_page", None) is not None + or config.unsafe_reset + ) + if not destructive: + return + if not config.allow_destructive: + raise HilFailure("destructive plan requires --allow-destructive") + if _upper(config.confirm_usb_serial) != EXPECTED_USB_SERIAL: + raise HilFailure( + "destructive plan requires --confirm-usb-serial " + f"{EXPECTED_USB_SERIAL}" + ) + + +def _list_serial_ports() -> List[Any]: + try: + from serial.tools import list_ports # type: ignore + except ImportError as exc: + raise HilFailure( + "pyserial is required; install it with 'python -m pip install " + "pyserial'" + ) from exc + return list(list_ports.comports()) + + +def resolve_initial_port( + requested_port: str, + port_source: Callable[[], Iterable[Any]] = _list_serial_ports, +) -> PortIdentity: + matches = [ + port_identity(item) + for item in port_source() + if _upper(getattr(item, "device", "")) == _upper(requested_port) + ] + if len(matches) != 1: + raise HilFailure( + f"expected exactly one enumerated {requested_port}, found {len(matches)}" + ) + validate_runtime_identity(matches[0], requested_port=requested_port) + return matches[0] + + +def _matching_runtime_ports( + expected_location: str, + port_source: Callable[[], Iterable[Any]], +) -> List[PortIdentity]: + matches: List[PortIdentity] = [] + for item in port_source(): + identity = port_identity(item) + try: + validate_runtime_identity( + identity, expected_location=expected_location + ) + except HilFailure: + continue + matches.append(identity) + return matches + + +async def wait_for_reenumeration( + previous: PortIdentity, + timeout: float, + port_source: Callable[[], Iterable[Any]] = _list_serial_ports, + error_future: Optional[asyncio.Future[Any]] = None, +) -> ReenumerationResult: + """Follow the identity through reset, including driver-retained devnodes.""" + + started = time.monotonic() + deadline = time.monotonic() + timeout + saw_absence = False + stable_device: Optional[str] = None + stable_samples = 0 + while time.monotonic() < deadline: + if error_future is not None and error_future.done(): + event = error_future.result() + raise RebootRefused(event) + matches = _matching_runtime_ports(previous.location, port_source) + if not matches: + saw_absence = True + stable_device = None + stable_samples = 0 + elif len(matches) > 1: + raise HilFailure( + "multiple matching T1000-E Companion ports appeared after reboot: " + + ", ".join(match.device for match in matches) + ) + else: + current = matches[0] + if _upper(current.device) == _upper(stable_device): + stable_samples += 1 + else: + stable_device = current.device + stable_samples = 1 + # Windows normally removes the devnode. Some CDC driver versions + # retain it across a USB stack reset, so also permit a stably + # identity-matched port after a bounded reset/recovery interval; + # reboot proof is completed with the device uptime check below. + recovery_elapsed = time.monotonic() - started + if stable_samples >= 2 and (saw_absence or recovery_elapsed >= 1.0): + return ReenumerationResult(current, saw_absence) + await asyncio.sleep(0.20) + raise HilFailure( + f"locked T1000-E did not return stably within {timeout:.1f} seconds" + ) + + +def _jsonable(value: Any) -> Any: + if isinstance(value, bytes): + return value.hex() + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if hasattr(value, "value"): + return _jsonable(value.value) + return value + + +class StressRunner: + def __init__(self, config: RunConfig, identity: PortIdentity) -> None: + self.config = config + self.identity = identity + self.mc: Any = None + self.steps: List[Dict[str, Any]] = [] + self.generated_keys: List[str] = [] + self.baseline_contact_count = 0 + self.baseline_contact_keys: set[str] = set() + self.expected_contact_keys: set[str] = set() + self.original_manual_add_contacts: Optional[bool] = None + self.original_autoadd_config: Optional[int] = None + self.original_channel_snapshot: Optional[Dict[str, Any]] = None + self.contact_enumeration_transactions = 0 + self.contact_enumeration_short_attempts = 0 + self.contact_enumeration_retries = 0 + self.last_contact_advertised_count: Optional[int] = None + + def progress(self, message: str) -> None: + if self.config.verbose: + print(message, file=sys.stderr, flush=True) + + def record(self, name: str, **details: Any) -> None: + self.steps.append({"step": name, **_jsonable(details)}) + self.progress(name) + + async def connect(self) -> None: + try: + from meshcore import MeshCore + except ImportError as exc: + raise HilFailure( + "meshcore Python package is required; install meshcore-cli" + ) from exc + self.mc = await MeshCore.create_serial( + self.identity.device, + baudrate=115200, + only_error=True, + default_timeout=8.0, + cx_dly=0.10, + ) + if self.mc is None: + raise HilFailure(f"APP_START failed on {self.identity.device}") + + self_event = await self.mc.commands.send_appstart() + self._require_event(self_event, "SELF_INFO") + node_key = _clean(self_event.payload.get("public_key", "")).lower() + expected_node_key = _clean( + self.config.expected_node_key_prefix + ).lower() + if expected_node_key and not node_key.startswith(expected_node_key): + raise IdentityGateFailure( + f"node public key starts {node_key[:8]!r}, expected requested " + f"prefix {expected_node_key}" + ) + + device_event = await self.mc.commands.send_device_query() + self._require_event(device_event, "DEVICE_INFO") + model = _clean(device_event.payload.get("model", "")) + if EXPECTED_MODEL_TOKEN.lower() not in model.lower(): + raise IdentityGateFailure( + f"firmware model is {model!r}, expected {EXPECTED_MODEL_TOKEN}" + ) + reported_capacity = int(device_event.payload.get("max_contacts", 0)) + if reported_capacity != self.config.contact_count: + raise IdentityGateFailure( + f"firmware reports {reported_capacity} contact slots, expected " + f"{self.config.contact_count}" + ) + reported_channels = int(device_event.payload.get("max_channels", 0)) + if not 0 <= self.config.channel_index < reported_channels: + raise IdentityGateFailure( + f"channel index {self.config.channel_index} is outside device " + f"capacity {reported_channels}" + ) + self.record( + "connected", + port=self.identity.device, + usb_serial=self.identity.serial_number, + node_key_prefix=node_key[:8], + device=device_event.payload, + ) + + async def disconnect(self) -> None: + if self.mc is None: + return + current, self.mc = self.mc, None + try: + await asyncio.wait_for(current.disconnect(), timeout=5.0) + except Exception as exc: + raise HilFailure(f"serial disconnect did not complete cleanly: {exc}") from exc + + async def connect_when_openable(self, timeout: float) -> None: + """Retry the brief Windows devnode-present-but-busy reset interval. + + A T1000-E CDC devnode can satisfy enumeration checks before the serial + driver will allow it to be opened. Retry transport/handshake failures, + but never retry an identity or capacity mismatch: that could silently + move a destructive run onto the wrong radio. + """ + + deadline = time.monotonic() + timeout + last_error: Optional[BaseException] = None + while True: + try: + await self.connect() + return + except IdentityGateFailure: + # connect() owns a live transport by the time application + # identity is checked. Drop it before failing closed so a + # comprehensive run's cleanup can never act on this radio. + try: + await self.disconnect() + except HilFailure: + self.mc = None + raise + except (OSError, asyncio.TimeoutError, HilFailure) as exc: + last_error = exc + try: + await self.disconnect() + except HilFailure: + # The reset may invalidate the handle while it is closing. + self.mc = None + remaining = deadline - time.monotonic() + if remaining <= 0: + break + await asyncio.sleep(min(0.25, remaining)) + + raise HilFailure( + f"identity-matched {self.identity.device} did not become openable " + f"within {timeout:.1f} seconds: {last_error}" + ) from last_error + + @staticmethod + def _require_event(event: Any, expected_name: str) -> Any: + event_name = getattr(getattr(event, "type", None), "name", "") + if event_name == "ERROR": + raise HilFailure( + f"expected {expected_name}, device returned error " + f"{getattr(event, 'payload', None)!r}" + ) + if event_name != expected_name: + raise HilFailure(f"expected {expected_name}, received {event_name!r}") + return event + + @staticmethod + def _require_error_code(event: Any, code: int, operation: str) -> None: + event_name = getattr(getattr(event, "type", None), "name", "") + payload = getattr(event, "payload", {}) + if event_name != "ERROR" or payload.get("error_code") != code: + raise HilFailure( + f"{operation} returned {event_name} {payload!r}, expected " + f"error code {code}" + ) + + async def cli( + self, command: str, *, timeout: Optional[float] = None + ) -> str: + if timeout is None: + event = await self.mc.commands.run_cli_command(command) + else: + # meshcore's run_cli_command() does not expose send()'s per-request + # timeout. Use the same public wire command directly so only the + # intentionally slow filler gets a longer deadline; keeping the + # ordinary 8-second timeout makes every other HIL failure prompt. + try: + from meshcore.events import EventType + from meshcore.packets import CommandType + except ImportError as exc: + raise HilFailure("meshcore Python package is unavailable") from exc + payload = bytes((CommandType.RUN_CLI_COMMAND.value,)) + command.encode( + "utf-8" + ) + event = await self.mc.commands.send( + payload, + [EventType.CLI_REPLY, EventType.ERROR], + timeout=timeout, + ) + self._require_event(event, "CLI_REPLY") + return _clean(event.payload.get("text", "")) + + async def clear_filler(self) -> None: + reply = await self.cli("hil extrafs clear") + if reply != "HIL ExtraFS filler cleared": + raise HilFailure(f"filler clear failed: {reply!r}") + + @staticmethod + def contact_key_token(public_key: str) -> str: + token = str(public_key)[:14].lower() + try: + decoded = bytes.fromhex(token) + except ValueError as exc: + raise HilFailure("contact key prefix is not hexadecimal") from exc + if len(decoded) != 7: + raise HilFailure("contact key prefix is not exactly seven bytes") + return token + + async def contact_slot(self, public_key: str) -> int: + token = self.contact_key_token(public_key) + return parse_contact_slot( + await self.cli(f"hil extrafs contact-slot {token}") + ) + + async def seed_cached_advert(self, public_key: str) -> int: + token = self.contact_key_token(public_key) + return parse_advert_seed_slot( + await self.cli(f"hil extrafs seed-advert {token} CONFIRM") + ) + + async def clear_cached_advert(self, public_key: str) -> None: + token = self.contact_key_token(public_key) + reply = await self.cli( + f"hil extrafs clear-advert {token} CONFIRM" + ) + if reply != "HIL advert cleared": + raise HilFailure(f"advert clear failed: {reply!r}") + + async def arm_contact_read_failure(self, page: int) -> None: + reply = await self.cli( + f"hil extrafs fail-read-page {page} CONFIRM" + ) + expected = f"HIL contact page {page} read failure armed" + if reply != expected: + raise HilFailure(f"contact read-failure arm failed: {reply!r}") + + async def arm_contact_stat_failure(self, page: int) -> None: + reply = await self.cli( + f"hil extrafs fail-stat-page {page} CONFIRM" + ) + expected = f"HIL contact page {page} stat failure armed" + if reply != expected: + raise HilFailure(f"contact stat-failure arm failed: {reply!r}") + + async def contact_page_mask(self, page: int) -> int: + parsed = parse_contact_page_mask( + await self.cli(f"hil extrafs page-mask {page}") + ) + if parsed.page != page: + raise HilFailure( + f"contact-page mask reply named page {parsed.page}, expected {page}" + ) + return parsed.occupied_mask + + async def wait_for_contact_page_mask( + self, page: int, expected_mask: int + ) -> int: + """Poll the persisted header until the lazy repair is durable.""" + + deadline = time.monotonic() + max( + self.config.settle_seconds, CONTACT_PAGE_REPAIR_TIMEOUT_SECONDS + ) + observed = await self.contact_page_mask(page) + while observed != expected_mask and time.monotonic() < deadline: + await asyncio.sleep(0.25) + observed = await self.contact_page_mask(page) + if observed != expected_mask: + raise HilFailure( + f"contact page {page} mask was not repaired: " + f"{observed:08x}, expected {expected_mask:08x}" + ) + return observed + + async def set_channel_exact( + self, channel_index: int, channel_name: str, channel_secret: bytes + ) -> Any: + """Set a channel without the client's '#' secret auto-derivation.""" + + try: + from meshcore.events import EventType + except ImportError as exc: + raise HilFailure("meshcore Python package is unavailable") from exc + name_bytes = channel_name.encode("utf-8")[:32].ljust(32, b"\0") + if len(channel_secret) != 16: + raise HilFailure("channel snapshot secret is not 16 bytes") + payload = ( + b"\x20" + + bytes((channel_index,)) + + name_bytes + + bytes(channel_secret) + ) + return await self.mc.commands.send( + payload, [EventType.OK, EventType.ERROR] + ) + + async def set_contact_exact(self, contact: Dict[str, Any]) -> Any: + """Add or update a contact while preserving its supplied lastmod.""" + + try: + from meshcore.events import EventType + except ImportError as exc: + raise HilFailure("meshcore Python package is unavailable") from exc + return await self.mc.commands.send( + encode_exact_contact_frame(contact), + [EventType.OK, EventType.ERROR], + ) + + async def core_uptime(self) -> int: + event = self._require_event( + await self.mc.commands.get_stats_core(), "STATS_CORE" + ) + return int(event.payload.get("uptime_secs", -1)) + + async def status(self) -> ExtraFsStatus: + status = parse_extrafs_status(await self.cli("hil extrafs status")) + if not status.active: + raise HilFailure("internal ExtraFS is not active") + if status.total_kib != EXPECTED_STORAGE_KIB: + raise HilFailure( + f"ExtraFS capacity is {status.total_kib} KiB, expected " + f"{EXPECTED_STORAGE_KIB} KiB" + ) + battery = self._require_event( + await self.mc.commands.get_bat(), "BATTERY" + ).payload + if ( + battery.get("used_kb") != status.used_kib + or battery.get("total_kb") != status.total_kib + ): + raise HilFailure( + "binary storage status disagrees with the HIL status reply" + ) + self.record("storage_status", hil=asdict(status), binary=battery) + return status + + async def _contact_snapshot_once( + self, + ) -> tuple[Dict[str, Dict[str, Any]], int]: + """Read one unfiltered stream and preserve its advertised count.""" + + reader = getattr(self.mc, "_reader", None) + if reader is None or not hasattr(reader, "contact_nb"): + raise HilFailure("client did not expose the CONTACTS_START count") + + # CONTACTS_START is not dispatched as an event by the current SDK. Clear + # its private aggregation state so a lost START cannot reuse the count or + # dictionary from the preceding pass and make a short stream look whole. + reader.contact_nb = None + reader.contacts = {} + self.contact_enumeration_transactions += 1 + event = await self.mc.commands.get_contacts(lastmod=0, timeout=20) + self._require_event(event, "CONTACTS") + contacts = event.payload + if not isinstance(contacts, dict): + raise HilFailure("CONTACTS payload is not a dictionary") + expected = getattr(reader, "contact_nb", None) + if not isinstance(expected, int) or expected < 0: + raise HilFailure("client did not expose a valid CONTACTS_START count") + + snapshot = dict(contacts) + self.last_contact_advertised_count = expected + if len(snapshot) != expected: + self.contact_enumeration_short_attempts += 1 + return snapshot, expected + + async def contacts(self) -> Dict[str, Dict[str, Any]]: + # Never use a short snapshot for destructive cleanup. The installed + # client records CONTACTS_START's unfiltered count but does not validate + # it before dispatching CONTACTS, so retry an incomplete transaction. + failures: List[str] = [] + for attempt in range(1, 4): + if attempt > 1: + self.contact_enumeration_retries += 1 + snapshot, expected = await self._contact_snapshot_once() + if len(snapshot) == expected: + return snapshot + + failures.append( + f"attempt {attempt}: expected {expected}, received {len(snapshot)}" + ) + self.progress(f"incomplete contact list; retrying ({failures[-1]})") + + raise HilFailure("incomplete contact enumeration; " + "; ".join(failures)) + + async def expect_contacts_file_io(self, context: str) -> None: + """Require GET_CONTACTS to fail before emitting any partial stream.""" + + reader = getattr(self.mc, "_reader", None) + if reader is None or not hasattr(reader, "contact_nb"): + raise HilFailure("client did not expose contact aggregation state") + reader.contact_nb = None + reader.contacts = {} + try: + from meshcore.events import EventType + except ImportError as exc: + raise HilFailure("meshcore Python package is unavailable") from exc + # The SDK's aggregate get_contacts() installs its ERROR waiter after a + # fire-and-forget send. This immediate firmware error can beat that + # waiter, so use send() directly to subscribe before transmitting. + event = await self.mc.commands.send( + b"\x04", [EventType.ERROR], timeout=20 + ) + self._require_error_code(event, 5, context) + if reader.contact_nb is not None or reader.contacts: + raise HilFailure( + f"{context}: device emitted partial contact stream state " + f"count={reader.contact_nb!r}, contacts={len(reader.contacts)}" + ) + + async def contact_by_key(self, public_key: str) -> Dict[str, Any]: + event = self._require_event( + await self.mc.commands.get_contact_by_key( + bytes.fromhex(public_key) + ), + "NEXT_CONTACT", + ) + contact = dict(event.payload) + if contact.get("public_key") != public_key: + raise HilFailure( + "single-contact query returned a different public key" + ) + return contact + + async def expect_contact_not_found( + self, public_key: str, context: str + ) -> None: + event = await self.mc.commands.get_contact_by_key( + bytes.fromhex(public_key) + ) + self._require_error_code(event, 2, context) + + async def stress_contact_enumerations( + self, + expected_keys: Optional[set[str]], + context: str, + ) -> Dict[str, Dict[str, Any]]: + """Repeat complete list reads and surface even successfully retried loss.""" + + requested = self.config.contact_enumeration_passes + if requested is None or requested < 1: + raise HilFailure("contact enumeration stress requires at least one pass") + + start_transactions = self.contact_enumeration_transactions + start_short = self.contact_enumeration_short_attempts + start_retries = self.contact_enumeration_retries + reference_keys = set(expected_keys) if expected_keys is not None else None + completed = 0 + final_snapshot: Dict[str, Dict[str, Any]] = {} + advertised_counts: List[int] = [] + received_counts: List[int] = [] + + def details(ok: bool, failure: Optional[str] = None) -> Dict[str, Any]: + keys = reference_keys if reference_keys is not None else set() + result: Dict[str, Any] = { + "context": context, + "ok": ok, + "requested_passes": requested, + "completed_passes": completed, + "transport_transactions": ( + self.contact_enumeration_transactions - start_transactions + ), + "short_attempts": ( + self.contact_enumeration_short_attempts - start_short + ), + "retry_attempts": ( + self.contact_enumeration_retries - start_retries + ), + "expected_count": len(keys), + "advertised_counts": advertised_counts, + "received_counts": received_counts, + "keyset_sha256": self.keyset_digest(keys), + "usb_serial": self.identity.serial_number, + } + if failure is not None: + result["failure"] = failure + return result + + try: + for pass_number in range(1, requested + 1): + final_snapshot = await self.contacts() + actual_keys = set(final_snapshot) + advertised_counts.append( + int(self.last_contact_advertised_count or 0) + ) + received_counts.append(len(final_snapshot)) + if reference_keys is None: + reference_keys = actual_keys + self.assert_exact_contacts( + final_snapshot, + reference_keys, + f"{context} pass {pass_number}/{requested}", + ) + completed = pass_number + self.progress( + f"contact enumeration pass {pass_number}/{requested}: " + f"{len(final_snapshot)} contacts" + ) + + expected_digest = self.config.expected_contact_keyset_sha256 + actual_digest = self.keyset_digest(reference_keys or set()) + if expected_digest is not None and actual_digest != expected_digest: + raise HilFailure( + f"{context}: keyset digest is {actual_digest}, expected " + f"{expected_digest}" + ) + + short_attempts = self.contact_enumeration_short_attempts - start_short + if short_attempts: + retry_attempts = self.contact_enumeration_retries - start_retries + raise HilFailure( + f"{context}: observed {short_attempts} incomplete contact " + f"stream(s) and {retry_attempts} retry attempt(s) across " + f"{requested} validated pass(es)" + ) + except BaseException as exc: + self.record("contact_enumeration_stress", **details(False, str(exc))) + raise + + self.record("contact_enumeration_stress", **details(True)) + return final_snapshot + + async def channels(self) -> List[Dict[str, Any]]: + device = self._require_event( + await self.mc.commands.send_device_query(), "DEVICE_INFO" + ).payload + maximum = int(device.get("max_channels", 0)) + if maximum <= 0: + raise HilFailure("device reported no channel capacity") + result = [] + for index in range(maximum): + payload = self._require_event( + await self.mc.commands.get_channel(index), "CHANNEL_INFO" + ).payload + result.append( + { + "channel_idx": payload["channel_idx"], + "channel_name": payload["channel_name"], + "channel_hash": payload["channel_hash"], + } + ) + return result + + async def set_manual_contact_mode(self, enabled: bool) -> None: + event = await self.mc.commands.set_manual_add_contacts(enabled) + self._require_event(event, "OK") + verify = self._require_event( + await self.mc.commands.send_appstart(), "SELF_INFO" + ).payload + if bool(verify.get("manual_add_contacts")) != enabled: + raise HilFailure("manual/automatic contact mode did not change") + self.record("manual_contact_mode", enabled=enabled) + + async def set_autoadd_config(self, value: int) -> None: + self._require_event( + await self.mc.commands.set_autoadd_config(value), "OK" + ) + verify = self._require_event( + await self.mc.commands.get_autoadd_config(), "AUTOADD_CONFIG" + ).payload + if int(verify.get("config", -1)) != value: + raise HilFailure( + f"automatic contact-type mask is {verify.get('config')}, " + f"expected {value}" + ) + self.record("autoadd_config", value=value) + + @staticmethod + def keyset_digest(keys: Iterable[str]) -> str: + canonical = "\n".join(sorted(keys)).encode("ascii") + return sha256(canonical).hexdigest() + + @staticmethod + def semantic_contact_digest(contacts: Dict[str, Dict[str, Any]]) -> str: + fields = ( + "public_key", + "type", + "flags", + "out_path_hash_mode", + "out_path_len", + "out_path", + "adv_name", + "last_advert", + "adv_lat", + "adv_lon", + "lastmod", + ) + normalized = [ + {field: contacts[key].get(field) for field in fields} + for key in sorted(contacts) + ] + canonical = json.dumps( + normalized, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("ascii") + return sha256(canonical).hexdigest() + + @staticmethod + def wire_contact_digest(contacts: Dict[str, Dict[str, Any]]) -> str: + digest = sha256() + for key in sorted(contacts): + digest.update(encode_exact_contact_frame(contacts[key])) + return digest.hexdigest() + + def exact_inventory_digests( + self, contacts: Dict[str, Dict[str, Any]] + ) -> Dict[str, str]: + return { + "keyset_sha256": self.keyset_digest(contacts), + "semantic_sha256": self.semantic_contact_digest(contacts), + "wire_sha256": self.wire_contact_digest(contacts), + } + + def assert_exact_contact_inventory( + self, + actual: Dict[str, Dict[str, Any]], + expected: Dict[str, Dict[str, Any]], + context: str, + ) -> Dict[str, str]: + self.assert_exact_contacts(actual, set(expected), context) + expected_digests = self.exact_inventory_digests(expected) + actual_digests = self.exact_inventory_digests(actual) + if actual_digests != expected_digests: + raise HilFailure( + f"{context}: exact inventory digest mismatch; " + f"actual={actual_digests}, expected={expected_digests}" + ) + return actual_digests + + async def capture_contact_slot_map( + self, + contacts: Dict[str, Dict[str, Any]], + context: str, + ) -> Dict[str, int]: + """Read every live persistent slot through the identity-gated HIL API.""" + + keys = list(contacts) + tokens = [self.contact_key_token(key) for key in keys] + if len(set(tokens)) != len(tokens): + raise HilFailure( + f"{context}: contact key prefixes are not unique enough for " + "identity-safe slot queries" + ) + + slots: Dict[str, int] = {} + owners: Dict[int, str] = {} + for key in keys: + slot = await self.contact_slot(key) + if slot in owners: + raise HilFailure( + f"{context}: contacts {owners[slot][:12]} and {key[:12]} " + f"both report persistent slot {slot}" + ) + slots[key] = slot + owners[slot] = key + return slots + + async def assert_contact_slot_map( + self, + keys: Iterable[str], + expected_slots: Dict[str, int], + context: str, + ) -> None: + """Require each named live contact to retain its captured slot.""" + + for key in keys: + expected = expected_slots.get(key) + if expected is None: + raise HilFailure( + f"{context}: no captured slot for contact {key[:12]}" + ) + actual = await self.contact_slot(key) + if actual != expected: + raise HilFailure( + f"{context}: contact {key[:12]} moved from slot " + f"{expected} to {actual}" + ) + + def assert_deterministic_contact_inventory( + self, contacts: Dict[str, Dict[str, Any]], context: str + ) -> set[str]: + deterministic_keys = { + make_contact(index)["public_key"] + for index in range(self.config.contact_count) + } + self.assert_exact_contacts(contacts, deterministic_keys, context) + if len({key[:14] for key in deterministic_keys}) != len( + deterministic_keys + ): + raise HilFailure("deterministic HIL contact prefixes are not unique") + for index in range(self.config.contact_count): + expected = make_contact(index) + key = expected["public_key"] + actual = contacts[key] + for field, expected_value in expected.items(): + if actual.get(field) != expected_value: + raise HilFailure( + f"{context}: {key[:12]} {field} is " + f"{actual.get(field)!r}, expected {expected_value!r}" + ) + return deterministic_keys + + def assert_exact_contacts( + self, + contacts: Dict[str, Dict[str, Any]], + expected_keys: set[str], + context: str, + ) -> None: + actual_keys = set(contacts) + if actual_keys != expected_keys: + missing = sorted(expected_keys - actual_keys) + unexpected = sorted(actual_keys - expected_keys) + raise HilFailure( + f"{context}: contact key set mismatch; " + f"missing={[key[:12] for key in missing[:8]]}, " + f"unexpected={[key[:12] for key in unexpected[:8]]}" + ) + + async def inspect(self) -> None: + await self.status() + if self.config.contact_enumeration_passes is not None: + contacts = await self.stress_contact_enumerations( + None, "read-only inventory" + ) + else: + contacts = await self.contacts() + channels = await self.channels() + self.record( + "inventory", + contact_count=len(contacts), + channels=channels, + ) + + async def add_contacts_to_capacity(self) -> set[str]: + existing = await self.contacts() + synthetic = [make_contact(index) for index in range(self.config.contact_count)] + synthetic_keys = {contact["public_key"] for contact in synthetic} + resumed_keys = set(existing) & synthetic_keys + self.baseline_contact_keys = set(existing) - synthetic_keys + self.baseline_contact_count = len(self.baseline_contact_keys) + if ( + getattr(self.config, "corrupt_occupied_page", None) is not None + and self.baseline_contact_keys + ): + raise HilFailure( + "occupied-mask recovery requires an existing inventory made " + "only from this harness's deterministic contacts" + ) + if existing and not self.config.allow_existing_contacts: + raise HilFailure( + f"expected an erased spare, but found {len(existing)} contact(s); " + "use --allow-existing-contacts only after reviewing that inventory" + ) + if len(existing) > self.config.contact_count: + raise HilFailure( + f"device already has {len(existing)} contacts, above requested " + f"capacity target {self.config.contact_count}" + ) + self.generated_keys = [ + contact["public_key"] + for contact in synthetic + if contact["public_key"] in resumed_keys + ] + to_add = self.config.contact_count - len(existing) + added = 0 + for contact in synthetic: + if added >= to_add: + break + if contact["public_key"] in existing: + continue + event = await self.mc.commands.add_contact(contact) + self._require_event(event, "OK") + self.generated_keys.append(contact["public_key"]) + added += 1 + if added % 25 == 0 or added == to_add: + self.progress(f"contacts added: {added}/{to_add}") + + if not self.generated_keys: + raise HilFailure( + "comprehensive contact mutation requires at least one " + "harness-owned deterministic contact" + ) + + await asyncio.sleep(self.config.settle_seconds) + persisted = await self.contacts() + self.expected_contact_keys = ( + self.baseline_contact_keys | set(self.generated_keys) + ) + if len(self.expected_contact_keys) != self.config.contact_count: + raise HilFailure("synthetic contact plan did not fill exactly 350 slots") + self.assert_exact_contacts( + persisted, self.expected_contact_keys, "after lazy persistence settle" + ) + overflow = make_contact(self.config.contact_count + 1000) + self._require_error_code( + await self.mc.commands.add_contact(overflow), + 3, + "contact beyond capacity", + ) + self.record( + "contact_capacity", + baseline=self.baseline_contact_count, + resumed=len(resumed_keys), + added=added, + generated_total=len(self.generated_keys), + persisted=len(persisted), + keyset_sha256=self.keyset_digest(persisted), + overflow_error="ERR_CODE_TABLE_FULL", + ) + return set(self.expected_contact_keys) + + async def fill_storage_to_enospc(self) -> ExtraFsFill: + """Create a verified, non-empty filler that exhausts ExtraFS.""" + + await self.clear_filler() + fill = parse_extrafs_fill( + await self.cli( + f"hil extrafs fill {self.config.fill_bytes}", + timeout=EXTRAFS_FILL_TIMEOUT_SECONDS, + ) + ) + if fill.requested_bytes != self.config.fill_bytes: + raise HilFailure("firmware echoed a different filler request size") + if fill.written_bytes >= fill.requested_bytes: + raise HilFailure( + "filler did not encounter ENOSPC; increase --fill-bytes" + ) + if fill.written_bytes <= 0: + raise HilFailure("filler did not commit any storage pressure") + if fill.total_kib != EXPECTED_STORAGE_KIB: + raise HilFailure("ExtraFS capacity changed during fill") + return fill + + async def fill_and_test_channel_atomicity( + self, + ) -> tuple[ExtraFsFill, Dict[str, Any]]: + before_event = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ) + before = before_event.payload + self.original_channel_snapshot = { + "channel_name": before.get("channel_name", ""), + "channel_secret": bytes(before.get("channel_secret", bytes(16))), + } + fill = await self.fill_storage_to_enospc() + + test_name = "HIL-FULL-ROLLBACK" + test_secret = sha256(test_name.encode()).digest()[:16] + set_event = await self.set_channel_exact( + self.config.channel_index, test_name, test_secret + ) + self._require_error_code( + set_event, 5, "channel write while ExtraFS is full" + ) + after = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ).payload + if ( + after.get("channel_name") != before.get("channel_name") + or after.get("channel_secret") != before.get("channel_secret") + ): + raise HilFailure("failed channel write changed the live channel table") + self.record( + "full_storage_atomicity", + fill=asdict(fill), + channel_index=self.config.channel_index, + expected_error="ERR_CODE_FILE_IO_ERROR", + rollback_verified=True, + ) + return fill, before + + async def reboot_and_reconnect(self, *, unsafe: bool = False) -> None: + old_identity = self.identity + before_uptime = await self.core_uptime() + if before_uptime < 6: + await asyncio.sleep(6 - max(before_uptime, 0)) + before_uptime = await self.core_uptime() + + loop = asyncio.get_running_loop() + reboot_error: asyncio.Future[Any] = loop.create_future() + + def capture_error(event: Any) -> None: + if not reboot_error.done(): + reboot_error.set_result(event) + + try: + from meshcore.events import EventType + except ImportError as exc: + raise HilFailure("meshcore Python package is unavailable") from exc + subscription = self.mc.subscribe(EventType.ERROR, capture_error) + if unsafe: + # Fire-and-forget: firmware resets before it can emit CLI_REPLY. + command = b"\x42hil extrafs unsafe-reset CONFIRM" + await self.mc.commands.send(command) + else: + # Register the ERROR subscription before sending: a dirty-page + # flush failure is the only response, while success resets without + # an acknowledgement. + await self.mc.commands.send(b"\x13reboot") + try: + transition = await wait_for_reenumeration( + old_identity, + self.config.reenum_timeout, + error_future=reboot_error, + ) + finally: + self.mc.unsubscribe(subscription) + try: + await self.disconnect() + except HilFailure: + # The device may have removed the OS handle before our close. + pass + self.identity = transition.identity + await self.connect_when_openable( + min(10.0, self.config.reenum_timeout) + ) + after_uptime = await self.core_uptime() + if not transition.observed_absence and after_uptime >= before_uptime: + raise HilFailure( + f"reopened device uptime did not reset: {before_uptime} -> " + f"{after_uptime} seconds" + ) + self.record( + "reboot_reenumeration", + unsafe=unsafe, + previous_port=old_identity.device, + current_port=self.identity.device, + location=self.identity.location, + observed_usb_absence=transition.observed_absence, + uptime_before=before_uptime, + uptime_after=after_uptime, + ) + + async def graceful_reboot_or_enospc_refusal( + self, + ) -> tuple[str, int, int]: + """Accept either a clean reboot or a verified FILE_IO refusal.""" + + before_uptime = await self.core_uptime() + if before_uptime < 6: + await asyncio.sleep(6 - max(before_uptime, 0)) + before_uptime = await self.core_uptime() + try: + await self.reboot_and_reconnect() + except RebootRefused as exc: + self._require_error_code( + exc.event, + 5, + "graceful reboot after rolled-back advert-backed removal", + ) + after_uptime = await self.core_uptime() + if after_uptime < before_uptime: + raise HilFailure( + "refused advert-rollback reboot reset uptime: " + f"{before_uptime} -> {after_uptime}" + ) + self.record( + "advert_rollback_reboot_refused", + error="ERR_CODE_FILE_IO_ERROR", + uptime_before=before_uptime, + uptime_after=after_uptime, + ) + return "refused", before_uptime, after_uptime + + # reboot_and_reconnect() already proves reset either by observing USB + # absence or (for retained devnodes) by comparing uptime internally. + # A slow Windows re-enumeration can legitimately make this later sample + # exceed the low pre-reset floor, so do not impose a second comparison. + after_uptime = await self.core_uptime() + return "rebooted", before_uptime, after_uptime + + async def verify_contacts_after_reboot( + self, expected_keys: set[str] + ) -> None: + if self.config.contact_enumeration_passes is not None: + contacts = await self.stress_contact_enumerations( + expected_keys, "after reboot" + ) + else: + contacts = await self.contacts() + self.assert_exact_contacts( + contacts, expected_keys, "after reboot" + ) + self.record( + "contact_reboot_persistence", + expected=len(expected_keys), + actual=len(contacts), + keyset_sha256=self.keyset_digest(contacts), + ) + + async def verify_full_state_after_reboot( + self, + expected_fill: ExtraFsFill, + expected_channel: Dict[str, Any], + ) -> None: + status = await self.status() + if status.filler_bytes != expected_fill.written_bytes: + raise HilFailure( + f"filler size after reboot is {status.filler_bytes}, expected " + f"{expected_fill.written_bytes}" + ) + if status.used_kib != expected_fill.used_kib: + raise HilFailure( + f"used space changed across reboot: {expected_fill.used_kib} -> " + f"{status.used_kib} KiB" + ) + channel = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ).payload + if ( + channel.get("channel_name") != expected_channel.get("channel_name") + or channel.get("channel_secret") + != expected_channel.get("channel_secret") + ): + raise HilFailure("failed full-filesystem channel write survived reboot") + self.record( + "full_storage_reboot_persistence", + filler_bytes=status.filler_bytes, + used_kib=status.used_kib, + channel_rollback_verified=True, + ) + + def assert_contact_variant( + self, + contacts: Dict[str, Dict[str, Any]], + expected_keys: set[str], + target_key: str, + expected_contact: Dict[str, Any], + context: str, + *, + include_lastmod: bool = False, + ) -> None: + self.assert_exact_contacts(contacts, expected_keys, context) + actual = contacts.get(target_key) + if actual is None: + raise HilFailure(f"{context}: target contact disappeared") + fields = [ + "public_key", + "type", + "flags", + "out_path_hash_mode", + "out_path_len", + "out_path", + "adv_name", + "last_advert", + "adv_lat", + "adv_lon", + ] + if include_lastmod: + fields.append("lastmod") + for field in fields: + if actual.get(field) != expected_contact.get(field): + raise HilFailure( + f"{context}: target {field} is {actual.get(field)!r}, " + f"expected {expected_contact.get(field)!r}" + ) + + @staticmethod + def assert_exact_contact_record( + actual: Dict[str, Any], expected: Dict[str, Any], context: str + ) -> None: + actual_wire = encode_exact_contact_frame(actual) + expected_wire = encode_exact_contact_frame(expected) + if actual_wire != expected_wire: + raise HilFailure(f"{context}: exact contact record changed") + + async def expect_enospc_reboot_refusal( + self, operation: str + ) -> tuple[int, int]: + """Prove a dirty contact page prevents a graceful ENOSPC reboot.""" + + # Give the normal lazy writer enough time to encounter ENOSPC before + # asking reboot to synchronously drain the same dirty page. + await asyncio.sleep(self.config.settle_seconds) + before_uptime = await self.core_uptime() + if before_uptime < 6: + await asyncio.sleep(6 - max(before_uptime, 0)) + before_uptime = await self.core_uptime() + try: + from meshcore.events import EventType + except ImportError as exc: + raise HilFailure("meshcore Python package is unavailable") from exc + reboot_event = await self.mc.commands.send( + b"\x13reboot", + [EventType.ERROR], + timeout=ENOSPC_REBOOT_REFUSAL_TIMEOUT_SECONDS, + ) + self._require_error_code(reboot_event, 5, operation) + after_uptime = await self.core_uptime() + if after_uptime < before_uptime: + raise HilFailure( + f"reboot refusal reset uptime: {before_uptime} -> " + f"{after_uptime}" + ) + return before_uptime, after_uptime + + async def restore_exact_contact_inventory( + self, + expected_keys: set[str], + original_contact: Dict[str, Any], + *, + replacement_key: Optional[str] = None, + context: str, + ) -> Dict[str, Dict[str, Any]]: + """Remove an owned substitute and durably restore one exact contact.""" + + await self.clear_filler() + original = dict(original_contact) + original_key = str(original["public_key"]) + current = await self.contacts() + if replacement_key is not None and replacement_key in current: + self._require_event( + await self.mc.commands.remove_contact(replacement_key), "OK" + ) + current = dict(current) + current.pop(replacement_key, None) + + self._require_event(await self.set_contact_exact(original), "OK") + + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + restored = await self.contacts() + self.assert_contact_variant( + restored, + expected_keys, + original_key, + original, + context, + include_lastmod=True, + ) + expected_digest = self.keyset_digest(expected_keys) + restored_digest = self.keyset_digest(restored) + if restored_digest != expected_digest: + raise HilFailure( + f"{context}: restored keyset digest is {restored_digest}, " + f"expected {expected_digest}" + ) + return restored + + async def test_contact_update_recovery_when_full( + self, expected_keys: set[str] + ) -> None: + """Prove a lazily ACKed contact cannot be lost to ENOSPC reboot.""" + + before_contacts = await self.contacts() + self.assert_exact_contacts( + before_contacts, expected_keys, "before full-storage contact update" + ) + preferred_key = make_contact(self.config.contact_count - 1)["public_key"] + safe_targets = [ + key for key in self.generated_keys if key in before_contacts + ] + if not safe_targets: + raise HilFailure( + "full-storage contact test found no harness-owned target; " + "refusing to mutate a pre-existing contact" + ) + target_key = ( + preferred_key if preferred_key in safe_targets else safe_targets[-1] + ) + original = dict(before_contacts[target_key]) + original.setdefault("public_key", target_key) + changed = dict(original) + changed["adv_name"] = ( + "HIL-ENOSPC-ALT" + if original.get("adv_name") == "HIL-ENOSPC" + else "HIL-ENOSPC" + ) + changed["flags"] = int(original.get("flags", 0)) ^ 0x01 + + async def restore_original() -> Dict[str, Dict[str, Any]]: + # Clear the test filler even if an earlier assertion failed. The + # contact page cannot be repaired while the intentional ENOSPC + # condition remains in place. + await self.clear_filler() + self._require_event( + await self.mc.commands.update_contact(dict(original)), "OK" + ) + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + restored_snapshot = await self.contacts() + self.assert_contact_variant( + restored_snapshot, + expected_keys, + target_key, + original, + "after restoring the full-storage contact", + ) + return restored_snapshot + + mutation_acknowledged = False + original_restored = False + try: + self._require_event( + await self.mc.commands.update_contact(changed), "OK" + ) + mutation_acknowledged = True + live_after_ack = await self.contacts() + self.assert_contact_variant( + live_after_ack, + expected_keys, + target_key, + changed, + "after full-storage contact ACK", + ) + + before_uptime, after_uptime = ( + await self.expect_enospc_reboot_refusal( + "graceful reboot with a dirty contact page at ENOSPC" + ) + ) + live_after_refusal = await self.contacts() + self.assert_contact_variant( + live_after_refusal, + expected_keys, + target_key, + changed, + "after refused full-storage reboot", + ) + + await self.clear_filler() + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + persisted = await self.contacts() + self.assert_contact_variant( + persisted, + expected_keys, + target_key, + changed, + "after freeing space and rebooting", + ) + + restored = await restore_original() + original_restored = True + except Exception as test_exc: + if mutation_acknowledged and not original_restored: + try: + await restore_original() + original_restored = True + self.record( + "full_storage_contact_emergency_restore", + target_key_prefix=target_key[:12], + original_restored=True, + ) + except Exception as restore_exc: + raise HilFailure( + f"full-storage contact test failed ({test_exc}); " + f"emergency restore also failed ({restore_exc})" + ) from test_exc + raise + self.record( + "full_storage_contact_recovery", + target_key_prefix=target_key[:12], + update_acknowledged=True, + reboot_error="ERR_CODE_FILE_IO_ERROR", + uptime_before_refusal=before_uptime, + uptime_after_refusal=after_uptime, + live_mutation_verified=True, + persisted_after_filler_clear=True, + original_restored=True, + keyset_sha256=self.keyset_digest(restored), + ) + + async def test_contact_add_recovery_when_full( + self, expected_keys: set[str] + ) -> None: + """Prove a lazily ACKed ADD is retained once ENOSPC is relieved.""" + + before_contacts = await self.contacts() + self.assert_exact_contacts( + before_contacts, expected_keys, "before full-storage contact add" + ) + safe_targets = [ + key for key in self.generated_keys if key in before_contacts + ] + preferred_key = make_contact(self.config.contact_count - 1)["public_key"] + if not safe_targets: + raise HilFailure( + "full-storage contact add test found no harness-owned target; " + "refusing to remove a pre-existing contact" + ) + original_key = ( + preferred_key if preferred_key in safe_targets else safe_targets[-1] + ) + original = dict(before_contacts[original_key]) + original.setdefault("public_key", original_key) + replacement = make_contact(self.config.contact_count + 2001) + replacement_key = replacement["public_key"] + if replacement_key in expected_keys: + raise HilFailure("full-storage ADD replacement already exists") + + without_original = set(expected_keys) - {original_key} + substituted_keys = without_original | {replacement_key} + mutation_attempted = False + original_restored = False + fill: Optional[ExtraFsFill] = None + try: + # Persist the empty slot before applying storage pressure. This + # makes the subsequent command an actual ADD, not an UPDATE. + mutation_attempted = True + self._require_event( + await self.mc.commands.remove_contact(original_key), "OK" + ) + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + persisted_empty_slot = await self.contacts() + self.assert_exact_contacts( + persisted_empty_slot, + without_original, + "after persisting the full-storage ADD test's empty slot", + ) + + fill = await self.fill_storage_to_enospc() + self._require_event( + await self.mc.commands.add_contact(dict(replacement)), "OK" + ) + live_after_ack = await self.contacts() + self.assert_contact_variant( + live_after_ack, + substituted_keys, + replacement_key, + replacement, + "after full-storage contact ADD ACK", + ) + + before_uptime, after_uptime = ( + await self.expect_enospc_reboot_refusal( + "graceful reboot with a dirty contact ADD at ENOSPC" + ) + ) + live_after_refusal = await self.contacts() + self.assert_contact_variant( + live_after_refusal, + substituted_keys, + replacement_key, + replacement, + "after refused full-storage contact ADD reboot", + ) + + await self.clear_filler() + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + persisted_replacement = await self.contacts() + self.assert_contact_variant( + persisted_replacement, + substituted_keys, + replacement_key, + replacement, + "after freeing space for the contact ADD", + ) + + restored = await self.restore_exact_contact_inventory( + expected_keys, + original, + replacement_key=replacement_key, + context="after restoring the full-storage ADD inventory", + ) + original_restored = True + except BaseException as test_exc: + if mutation_attempted and not original_restored: + try: + restored = await self.restore_exact_contact_inventory( + expected_keys, + original, + replacement_key=replacement_key, + context=( + "after emergency restoration of the full-storage " + "ADD inventory" + ), + ) + original_restored = True + self.record( + "full_storage_contact_add_emergency_restore", + original_key_prefix=original_key[:12], + replacement_key_prefix=replacement_key[:12], + original_restored=True, + keyset_sha256=self.keyset_digest(restored), + ) + except BaseException as restore_exc: + raise HilFailure( + f"full-storage contact ADD test failed ({test_exc}); " + f"emergency restore also failed ({restore_exc})" + ) from test_exc + raise + + self.record( + "full_storage_contact_add_recovery", + original_key_prefix=original_key[:12], + replacement_key_prefix=replacement_key[:12], + fill=asdict(fill), + add_acknowledged=True, + reboot_error="ERR_CODE_FILE_IO_ERROR", + uptime_before_refusal=before_uptime, + uptime_after_refusal=after_uptime, + live_mutation_verified=True, + persisted_after_filler_clear=True, + original_restored=True, + keyset_sha256=self.keyset_digest(restored), + ) + + async def test_contact_remove_recovery_when_full( + self, expected_keys: set[str] + ) -> None: + """Prove a lazily ACKed REMOVE is retained once ENOSPC is relieved.""" + + before_contacts = await self.contacts() + self.assert_exact_contacts( + before_contacts, expected_keys, "before full-storage contact remove" + ) + safe_targets = [ + key for key in self.generated_keys if key in before_contacts + ] + preferred_key = make_contact(self.config.contact_count - 2)["public_key"] + if not safe_targets: + raise HilFailure( + "full-storage contact remove test found no harness-owned target; " + "refusing to remove a pre-existing contact" + ) + target_key = ( + preferred_key if preferred_key in safe_targets else safe_targets[0] + ) + original = dict(before_contacts[target_key]) + original.setdefault("public_key", target_key) + removed_keys = set(expected_keys) - {target_key} + + mutation_attempted = False + original_restored = False + try: + # Prime a verified tombstone while space is available. Removing + # this synthetic contact at ENOSPC then takes the no-write cache + # path instead of needing to create a new bucket tombstone. + await self.clear_cached_advert(target_key) + fill = await self.fill_storage_to_enospc() + mutation_attempted = True + self._require_event( + await self.mc.commands.remove_contact(target_key), "OK" + ) + live_after_ack = await self.contacts() + self.assert_exact_contacts( + live_after_ack, + removed_keys, + "after full-storage contact REMOVE ACK", + ) + + before_uptime, after_uptime = ( + await self.expect_enospc_reboot_refusal( + "graceful reboot with a dirty contact REMOVE at ENOSPC" + ) + ) + live_after_refusal = await self.contacts() + self.assert_exact_contacts( + live_after_refusal, + removed_keys, + "after refused full-storage contact REMOVE reboot", + ) + + await self.clear_filler() + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + persisted_removal = await self.contacts() + self.assert_exact_contacts( + persisted_removal, + removed_keys, + "after freeing space for the contact REMOVE", + ) + + restored = await self.restore_exact_contact_inventory( + expected_keys, + original, + context="after restoring the full-storage REMOVE inventory", + ) + original_restored = True + except BaseException as test_exc: + if mutation_attempted and not original_restored: + try: + restored = await self.restore_exact_contact_inventory( + expected_keys, + original, + context=( + "after emergency restoration of the full-storage " + "REMOVE inventory" + ), + ) + original_restored = True + self.record( + "full_storage_contact_remove_emergency_restore", + target_key_prefix=target_key[:12], + original_restored=True, + keyset_sha256=self.keyset_digest(restored), + ) + except BaseException as restore_exc: + raise HilFailure( + f"full-storage contact REMOVE test failed ({test_exc}); " + f"emergency restore also failed ({restore_exc})" + ) from test_exc + else: + try: + await self.clear_filler() + except BaseException as clear_exc: + raise HilFailure( + f"full-storage contact REMOVE test failed ({test_exc}); " + f"emergency filler clear also failed ({clear_exc})" + ) from test_exc + raise + + self.record( + "full_storage_contact_remove_recovery", + target_key_prefix=target_key[:12], + fill=asdict(fill), + remove_acknowledged=True, + reboot_error="ERR_CODE_FILE_IO_ERROR", + uptime_before_refusal=before_uptime, + uptime_after_refusal=after_uptime, + live_mutation_verified=True, + persisted_after_filler_clear=True, + original_restored=True, + keyset_sha256=self.keyset_digest(restored), + ) + + async def restore_advert_rollback_inventory( + self, + baseline: Dict[str, Dict[str, Any]], + earlier_key: str, + target_key: str, + earlier_slot: int, + target_slot: int, + *, + context: str, + ) -> tuple[Dict[str, Dict[str, Any]], Dict[str, str]]: + """Clear advert/filler artifacts and restore both original slots.""" + + await self.clear_filler() + current = await self.contacts() + baseline_keys = set(baseline) + unexpected = set(current) - baseline_keys + missing = baseline_keys - set(current) + if unexpected or not missing <= {earlier_key, target_key}: + raise HilFailure( + f"{context}: refusing ambiguous recovery; " + f"missing={[key[:12] for key in sorted(missing)]}, " + f"unexpected={[key[:12] for key in sorted(unexpected)]}" + ) + + # Restore in ascending slot order. If an unexpected OK or lost reply + # removed both contacts, this makes addContact() reclaim the same two + # holes instead of swapping their persistent slots. + target_was_present = target_key in current + if target_was_present: + await self.clear_cached_advert(target_key) + self._require_event( + await self.set_contact_exact(dict(baseline[earlier_key])), "OK" + ) + self._require_event( + await self.set_contact_exact(dict(baseline[target_key])), "OK" + ) + if not target_was_present: + await self.clear_cached_advert(target_key) + + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + restored = await self.contacts() + digests = self.assert_exact_contact_inventory( + restored, baseline, context + ) + restored_earlier_slot = await self.contact_slot(earlier_key) + restored_target_slot = await self.contact_slot(target_key) + if ( + restored_earlier_slot != earlier_slot + or restored_target_slot != target_slot + ): + raise HilFailure( + f"{context}: slots changed from {earlier_slot}/{target_slot} " + f"to {restored_earlier_slot}/{restored_target_slot}" + ) + return restored, digests + + async def test_advert_backed_remove_rollback( + self, expected_keys: set[str] + ) -> None: + """Prove advert-cache ENOSPC removal rollback is slot-exact.""" + + baseline = await self.contacts() + deterministic_keys = self.assert_deterministic_contact_inventory( + baseline, + "before advert-backed contact removal rollback", + ) + if expected_keys != deterministic_keys: + raise HilFailure( + "advert rollback requires the exact deterministic 350-key set" + ) + first_key = make_contact(0)["public_key"] + last_key = make_contact(self.config.contact_count - 1)["public_key"] + first_slot = await self.contact_slot(first_key) + last_slot = await self.contact_slot(last_key) + if first_slot == last_slot: + raise HilFailure("two deterministic contacts reported the same slot") + if first_slot < last_slot: + earlier_key, earlier_slot = first_key, first_slot + target_key, target_slot = last_key, last_slot + else: + earlier_key, earlier_slot = last_key, last_slot + target_key, target_slot = first_key, first_slot + + baseline_digests = self.assert_exact_contact_inventory( + baseline, baseline, "capturing advert rollback baseline" + ) + without_earlier = { + key: dict(contact) + for key, contact in baseline.items() + if key != earlier_key + } + mutation_attempted = False + original_restored = False + try: + mutation_attempted = True + self._require_event( + await self.mc.commands.remove_contact(earlier_key), "OK" + ) + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + persisted_hole = await self.contacts() + self.assert_exact_contact_inventory( + persisted_hole, + without_earlier, + "after persisting the earlier advert-rollback hole", + ) + if await self.contact_slot(target_key) != target_slot: + raise HilFailure("later target moved while persisting earlier hole") + + seeded_slot = await self.seed_cached_advert(target_key) + if seeded_slot != target_slot: + raise HilFailure( + f"advert seeded in slot {seeded_slot}, expected {target_slot}" + ) + fill = await self.fill_storage_to_enospc() + + self._require_error_code( + await self.mc.commands.remove_contact(target_key), + 5, + "advert-backed contact removal at ENOSPC", + ) + live_after_rollback = await self.contacts() + self.assert_exact_contact_inventory( + live_after_rollback, + without_earlier, + "after failed advert-backed contact removal", + ) + if await self.contact_slot(target_key) != target_slot: + raise HilFailure("failed removal changed the target slot") + + reboot_outcome, uptime_before, uptime_after = ( + await self.graceful_reboot_or_enospc_refusal() + ) + after_reboot_attempt = await self.contacts() + self.assert_exact_contact_inventory( + after_reboot_attempt, + without_earlier, + "after advert-rollback reboot attempt", + ) + if await self.contact_slot(target_key) != target_slot: + raise HilFailure("reboot attempt changed the rolled-back target slot") + + restored, restored_digests = ( + await self.restore_advert_rollback_inventory( + baseline, + earlier_key, + target_key, + earlier_slot, + target_slot, + context="after restoring advert rollback baseline", + ) + ) + original_restored = True + except BaseException as test_exc: + if mutation_attempted and not original_restored: + try: + restored, restored_digests = ( + await self.restore_advert_rollback_inventory( + baseline, + earlier_key, + target_key, + earlier_slot, + target_slot, + context=( + "after emergency advert rollback restoration" + ), + ) + ) + original_restored = True + self.record( + "advert_remove_rollback_emergency_restore", + earlier_key_prefix=earlier_key[:12], + target_key_prefix=target_key[:12], + original_restored=True, + **restored_digests, + ) + except BaseException as restore_exc: + raise HilFailure( + f"advert-backed removal rollback failed ({test_exc}); " + f"emergency restore also failed ({restore_exc})" + ) from test_exc + raise + + if restored_digests != baseline_digests: + raise HilFailure( + "advert rollback restoration digest changed despite exact " + f"inventory verification: {restored_digests} != " + f"{baseline_digests}" + ) + self.record( + "advert_remove_rollback", + earlier_key_prefix=earlier_key[:12], + earlier_slot=earlier_slot, + target_key_prefix=target_key[:12], + target_slot=target_slot, + seeded_advert_slot=seeded_slot, + fill=asdict(fill), + remove_error="ERR_CODE_FILE_IO_ERROR", + live_target_and_slot_rolled_back=True, + reboot_outcome=reboot_outcome, + uptime_before_reboot_attempt=uptime_before, + uptime_after_reboot_attempt=uptime_after, + original_restored=True, + **restored_digests, + ) + + async def select_read_failure_contacts( + self, page: int + ) -> tuple[str, str, str, Dict[str, int]]: + """Choose one quarantined-page key and two readable-page keys.""" + + likely = [ + page * EXPECTED_CONTACTS_PER_PAGE, + page * EXPECTED_CONTACTS_PER_PAGE + 1, + 0, + 1, + self.config.contact_count - 2, + self.config.contact_count - 1, + ] + ordered_indexes = list(dict.fromkeys( + likely + list(range(self.config.contact_count)) + )) + failed_key: Optional[str] = None + available: List[str] = [] + slots: Dict[str, int] = {} + for index in ordered_indexes: + key = make_contact(index)["public_key"] + slot = await self.contact_slot(key) + if slot // EXPECTED_CONTACTS_PER_PAGE == page: + if failed_key is None: + failed_key = key + slots[key] = slot + elif len(available) < 2: + available.append(key) + slots[key] = slot + if failed_key is not None and len(available) == 2: + break + if failed_key is None or len(available) != 2: + raise HilFailure( + f"could not select read-failure contacts for page {page}" + ) + return failed_key, available[0], available[1], slots + + async def recover_read_failure_baseline( + self, + baseline: Dict[str, Dict[str, Any]], + expected_slots: Dict[str, int], + *, + context: str, + ) -> tuple[Dict[str, Dict[str, Any]], Dict[str, str]]: + """Consume any pending marker/incomplete boot and prove exact recovery.""" + + await self.clear_filler() + # Two reboots cover both ambiguous states after a lost host reply: + # marker armed but not consumed, or marker consumed and load incomplete. + await self.reboot_and_reconnect() + await self.reboot_and_reconnect() + recovered = await self.contacts() + digests = self.assert_exact_contact_inventory( + recovered, baseline, context + ) + for key, expected_slot in expected_slots.items(): + actual_slot = await self.contact_slot(key) + if actual_slot != expected_slot: + raise HilFailure( + f"{context}: {key[:12]} moved from slot " + f"{expected_slot} to {actual_slot}" + ) + status = await self.status() + if status.filler_bytes != 0: + raise HilFailure( + f"{context}: filler remains at {status.filler_bytes} bytes" + ) + return recovered, digests + + async def test_contact_page_read_failure(self, page: int) -> None: + """Prove one unread page fails closed and recovers untouched.""" + + await self.clear_filler() + initial_status = await self.status() + if initial_status.filler_bytes != 0: + raise HilFailure("read-failure test requires a clear filler") + baseline = await self.contacts() + self.assert_deterministic_contact_inventory( + baseline, "before contact-page read-failure injection" + ) + baseline_digests = self.assert_exact_contact_inventory( + baseline, baseline, "capturing contact read-failure baseline" + ) + if await self.contact_page_mask(page) != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"contact read-failure page {page} is not fully populated" + ) + failed_key, update_key, remove_key, expected_slots = ( + await self.select_read_failure_contacts(page) + ) + + marker_attempted = False + recovered_exactly = False + try: + marker_attempted = True + await self.arm_contact_read_failure(page) + await self.reboot_and_reconnect() + + await self.expect_contacts_file_io( + "GET_CONTACTS with one unread persisted page" + ) + await self.expect_contact_not_found( + failed_key, "contact from quarantined page" + ) + update_original = dict(baseline[update_key]) + remove_original = dict(baseline[remove_key]) + self.assert_exact_contact_record( + await self.contact_by_key(update_key), + update_original, + "available update target before blocked mutations", + ) + self.assert_exact_contact_record( + await self.contact_by_key(remove_key), + remove_original, + "available remove target before blocked mutations", + ) + + add_candidate = make_contact(self.config.contact_count + 3001) + add_candidate["lastmod"] = max( + int(contact["lastmod"]) for contact in baseline.values() + ) + 1 + await self.expect_contact_not_found( + add_candidate["public_key"], + "new contact before blocked ADD", + ) + self._require_error_code( + await self.set_contact_exact(add_candidate), + 5, + "contact ADD with an unread persisted page", + ) + await self.expect_contact_not_found( + add_candidate["public_key"], + "new contact after blocked ADD", + ) + + update_candidate = dict(update_original) + update_candidate["adv_name"] = "HIL-READFAIL-UPD" + update_candidate["flags"] = int(update_candidate["flags"]) ^ 1 + self._require_error_code( + await self.set_contact_exact(update_candidate), + 5, + "contact UPDATE with an unread persisted page", + ) + self.assert_exact_contact_record( + await self.contact_by_key(update_key), + update_original, + "available update target after blocked UPDATE", + ) + update_slot_after = await self.contact_slot(update_key) + if update_slot_after != expected_slots[update_key]: + raise HilFailure( + "blocked contact UPDATE moved the live target from slot " + f"{expected_slots[update_key]} to {update_slot_after}" + ) + + self._require_error_code( + await self.mc.commands.remove_contact(remove_key), + 5, + "contact REMOVE with an unread persisted page", + ) + self.assert_exact_contact_record( + await self.contact_by_key(remove_key), + remove_original, + "available remove target after blocked REMOVE", + ) + remove_slot_after = await self.contact_slot(remove_key) + if remove_slot_after != expected_slots[remove_key]: + raise HilFailure( + "blocked contact REMOVE moved the live target from slot " + f"{expected_slots[remove_key]} to {remove_slot_after}" + ) + self.assert_exact_contact_record( + await self.contact_by_key(update_key), + update_original, + "available update target after all blocked mutations", + ) + await self.expect_contacts_file_io( + "GET_CONTACTS after blocked unread-page mutations" + ) + incomplete_status = await self.status() + if incomplete_status.filler_bytes != 0: + raise HilFailure("read-failure boot unexpectedly created filler") + + # The marker was consumed during the incomplete boot, so one clean + # reboot must recover the untouched source page. + await self.reboot_and_reconnect() + recovered = await self.contacts() + recovered_digests = self.assert_exact_contact_inventory( + recovered, + baseline, + "after contact-page read-failure recovery reboot", + ) + for key, expected_slot in expected_slots.items(): + actual_slot = await self.contact_slot(key) + if actual_slot != expected_slot: + raise HilFailure( + f"read-failure recovery moved {key[:12]} from slot " + f"{expected_slot} to {actual_slot}" + ) + final_status = await self.status() + if final_status.filler_bytes != 0: + raise HilFailure( + "contact read-failure recovery left filler behind" + ) + if final_status.used_kib != initial_status.used_kib: + raise HilFailure( + "contact read-failure recovery changed ExtraFS usage from " + f"{initial_status.used_kib} KiB to " + f"{final_status.used_kib} KiB" + ) + recovered_exactly = True + except BaseException as test_exc: + if marker_attempted and not recovered_exactly: + try: + recovered, recovered_digests = ( + await self.recover_read_failure_baseline( + baseline, + expected_slots, + context=( + "after emergency contact read-failure recovery" + ), + ) + ) + recovered_exactly = True + self.record( + "contact_read_failure_emergency_recovery", + page=page, + recovered_exactly=True, + **recovered_digests, + ) + except BaseException as recover_exc: + raise HilFailure( + f"contact read-failure test failed ({test_exc}); " + f"emergency recovery also failed ({recover_exc})" + ) from test_exc + raise + + if recovered_digests != baseline_digests: + raise HilFailure( + "contact read-failure recovery changed exact inventory digests" + ) + self.record( + "contact_read_failure_recovery", + page=page, + failed_key_prefix=failed_key[:12], + update_key_prefix=update_key[:12], + remove_key_prefix=remove_key[:12], + expected_slots={key[:12]: slot for key, slot in expected_slots.items()}, + get_contacts_error="ERR_CODE_FILE_IO_ERROR", + add_error="ERR_CODE_FILE_IO_ERROR", + update_error="ERR_CODE_FILE_IO_ERROR", + remove_error="ERR_CODE_FILE_IO_ERROR", + partial_inventory_rejected=True, + live_mutations_rolled_back=True, + live_slots_rolled_back=True, + recovered_exactly=True, + filler_cleared=True, + baseline_used_kib=initial_status.used_kib, + final_used_kib=final_status.used_kib, + **recovered_digests, + ) + + async def test_contact_page_stat_failure(self, page: int) -> None: + """Prove a first-pass page stat error blocks the whole store safely.""" + + await self.clear_filler() + initial_status = await self.status() + if initial_status.filler_bytes != 0: + raise HilFailure("stat-failure test requires a clear filler") + baseline = await self.contacts() + self.assert_deterministic_contact_inventory( + baseline, "before contact-page stat-failure injection" + ) + baseline_digests = self.assert_exact_contact_inventory( + baseline, baseline, "capturing contact stat-failure baseline" + ) + if await self.contact_page_mask(page) != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"contact stat-failure page {page} is not fully populated" + ) + failed_key, update_key, remove_key, expected_slots = ( + await self.select_read_failure_contacts(page) + ) + + marker_attempted = False + recovered_exactly = False + try: + marker_attempted = True + await self.arm_contact_stat_failure(page) + await self.reboot_and_reconnect() + + await self.expect_contacts_file_io( + "GET_CONTACTS after contact-page presence-stat failure" + ) + add_candidate = make_contact(self.config.contact_count + 4001) + add_candidate["lastmod"] = max( + int(contact["lastmod"]) for contact in baseline.values() + ) + 1 + self._require_error_code( + await self.set_contact_exact(add_candidate), + 5, + "contact ADD with incomplete source discovery", + ) + + update_candidate = dict(baseline[update_key]) + update_candidate["adv_name"] = "HIL-STATFAIL-UPD" + update_candidate["flags"] = int(update_candidate["flags"]) ^ 1 + self._require_error_code( + await self.set_contact_exact(update_candidate), + 5, + "contact UPDATE with incomplete source discovery", + ) + self._require_error_code( + await self.mc.commands.remove_contact(remove_key), + 5, + "contact REMOVE with incomplete source discovery", + ) + await self.expect_contacts_file_io( + "GET_CONTACTS after blocked stat-failure mutations" + ) + incomplete_status = await self.status() + if incomplete_status.filler_bytes != 0: + raise HilFailure("stat-failure boot unexpectedly created filler") + if incomplete_status.used_kib != initial_status.used_kib: + raise HilFailure( + "contact stat-failure boot changed ExtraFS usage from " + f"{initial_status.used_kib} KiB to " + f"{incomplete_status.used_kib} KiB" + ) + + # The high-bit marker was consumed before source discovery, so the + # untouched page and complete table must return after one reboot. + await self.reboot_and_reconnect() + recovered = await self.contacts() + recovered_digests = self.assert_exact_contact_inventory( + recovered, + baseline, + "after contact-page stat-failure recovery reboot", + ) + for key, expected_slot in expected_slots.items(): + actual_slot = await self.contact_slot(key) + if actual_slot != expected_slot: + raise HilFailure( + f"stat-failure recovery moved {key[:12]} from slot " + f"{expected_slot} to {actual_slot}" + ) + final_status = await self.status() + if final_status.filler_bytes != 0: + raise HilFailure( + "contact stat-failure recovery left filler behind" + ) + if final_status.used_kib != initial_status.used_kib: + raise HilFailure( + "contact stat-failure recovery changed ExtraFS usage from " + f"{initial_status.used_kib} KiB to " + f"{final_status.used_kib} KiB" + ) + recovered_exactly = True + except BaseException as test_exc: + if marker_attempted and not recovered_exactly: + try: + recovered, recovered_digests = ( + await self.recover_read_failure_baseline( + baseline, + expected_slots, + context=( + "after emergency contact stat-failure recovery" + ), + ) + ) + recovered_exactly = True + self.record( + "contact_stat_failure_emergency_recovery", + page=page, + recovered_exactly=True, + **recovered_digests, + ) + except BaseException as recover_exc: + raise HilFailure( + f"contact stat-failure test failed ({test_exc}); " + f"emergency recovery also failed ({recover_exc})" + ) from test_exc + raise + + if recovered_digests != baseline_digests: + raise HilFailure( + "contact stat-failure recovery changed exact inventory digests" + ) + self.record( + "contact_stat_failure_recovery", + page=page, + failed_key_prefix=failed_key[:12], + update_key_prefix=update_key[:12], + remove_key_prefix=remove_key[:12], + expected_slots={key[:12]: slot for key, slot in expected_slots.items()}, + get_contacts_error="ERR_CODE_FILE_IO_ERROR", + add_error="ERR_CODE_FILE_IO_ERROR", + update_error="ERR_CODE_FILE_IO_ERROR", + remove_error="ERR_CODE_FILE_IO_ERROR", + whole_source_discovery_rejected=True, + recovered_exactly=True, + marker_storage_reclaimed=True, + baseline_used_kib=initial_status.used_kib, + final_used_kib=final_status.used_kib, + **recovered_digests, + ) + + async def test_channel_persistence_when_free(self) -> None: + await self.clear_filler() + before = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ).payload + if self.original_channel_snapshot is None: + self.original_channel_snapshot = { + "channel_name": before.get("channel_name", ""), + "channel_secret": bytes( + before.get("channel_secret", bytes(16)) + ), + } + test_name = "HIL-FREE-COMMIT" + test_secret = sha256(test_name.encode()).digest()[:16] + self._require_event( + await self.set_channel_exact( + self.config.channel_index, test_name, test_secret + ), + "OK", + ) + await self.reboot_and_reconnect() + committed = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ).payload + if ( + committed.get("channel_name") != test_name + or committed.get("channel_secret") != test_secret + ): + raise HilFailure("channel update did not survive reboot") + + original_name = before.get("channel_name", "") + original_secret = before.get("channel_secret", bytes(16)) + self._require_event( + await self.set_channel_exact( + self.config.channel_index, original_name, original_secret + ), + "OK", + ) + await self.reboot_and_reconnect() + restored = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ).payload + if ( + restored.get("channel_name") != original_name + or restored.get("channel_secret") != original_secret + ): + raise HilFailure("original channel did not survive restore reboot") + self.record( + "channel_commit_and_restore", + channel_index=self.config.channel_index, + persistence_verified=True, + ) + + async def restore_contact_inventory_exact( + self, + baseline: Dict[str, Dict[str, Any]], + baseline_order: List[str], + expected_slots: Dict[str, int], + page: int, + *, + current: Optional[Dict[str, Dict[str, Any]]] = None, + context: str, + ) -> tuple[Dict[str, Dict[str, Any]], Dict[str, str]]: + """Restore a baseline subset into its original lowest-free slots.""" + + if ( + len(baseline_order) != len(baseline) + or set(baseline_order) != set(baseline) + or set(expected_slots) != set(baseline) + ): + raise HilFailure(f"{context}: incomplete baseline restoration metadata") + if current is None: + current = await self.contacts() + else: + current = {key: dict(value) for key, value in current.items()} + + unexpected = set(current) - set(baseline) + if unexpected: + raise HilFailure( + f"{context}: cannot restore around unexpected contacts " + f"{[key[:12] for key in sorted(unexpected)[:8]]}" + ) + expected_current = { + key: baseline[key] + for key in current + } + self.assert_exact_contact_inventory( + current, expected_current, f"{context}: surviving contacts" + ) + expected_current_order = [ + key for key in baseline_order if key in current + ] + if list(current) != expected_current_order: + raise HilFailure( + f"{context}: surviving contacts are not in captured slot order" + ) + await self.assert_contact_slot_map( + current, expected_slots, f"{context}: surviving slots" + ) + + missing = sorted( + set(baseline) - set(current), key=expected_slots.__getitem__ + ) + for key in missing: + self._require_event( + await self.set_contact_exact(dict(baseline[key])), "OK" + ) + actual_slot = await self.contact_slot(key) + expected_slot = expected_slots[key] + if actual_slot != expected_slot: + raise HilFailure( + f"{context}: restored contact {key[:12]} entered slot " + f"{actual_slot}, expected {expected_slot}" + ) + + await asyncio.sleep(self.config.settle_seconds) + if await self.contact_page_mask(page) != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"{context}: restored page {page} is not fully occupied" + ) + + # The first reboot proves the rebuilt page was committed. Query every + # slot afterward rather than inferring slot identity from key equality. + await self.reboot_and_reconnect() + first = await self.contacts() + digests = self.assert_exact_contact_inventory( + first, baseline, f"{context}: after restore reboot" + ) + if list(first) != baseline_order: + raise HilFailure( + f"{context}: persistent contact stream order changed after restore" + ) + await self.assert_contact_slot_map( + first, expected_slots, f"{context}: restored slot map" + ) + if await self.contact_page_mask(page) != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"{context}: full restored page {page} did not survive reboot" + ) + + # A second reboot distinguishes a durable rebuild from a RAM-only view + # and rechecks the exact semantic/wire inventory and stream ordering. + await self.reboot_and_reconnect() + durable = await self.contacts() + durable_digests = self.assert_exact_contact_inventory( + durable, baseline, f"{context}: after durability reboot" + ) + if list(durable) != baseline_order: + raise HilFailure( + f"{context}: persistent stream order changed on durability reboot" + ) + if await self.contact_page_mask(page) != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"{context}: restored page {page} mask was not durable" + ) + if durable_digests != digests: + raise HilFailure( + f"{context}: exact inventory digests changed across reboots" + ) + return durable, durable_digests + + async def corrupt_contact_page( + self, page: int, before_contacts: Dict[str, Dict[str, Any]] + ) -> set[str]: + """Prove CRC-loss isolation, then restore exact records and slots.""" + + await self.clear_filler() + initial_status = await self.status() + if initial_status.filler_bytes != 0: + raise HilFailure("contact-page corruption requires a clear filler") + + baseline = { + key: dict(value) for key, value in before_contacts.items() + } + baseline_order = list(baseline) + if len(baseline) != self.config.contact_count: + raise HilFailure( + "restorative page corruption requires a full 350-contact table" + ) + baseline_digests = self.assert_exact_contact_inventory( + baseline, baseline, "capturing contact-page corruption baseline" + ) + expected_keyset = getattr( + self.config, "expected_contact_keyset_sha256", None + ) + if ( + expected_keyset is not None + and baseline_digests["keyset_sha256"] != expected_keyset + ): + raise HilFailure( + "contact-page corruption baseline keyset digest is " + f"{baseline_digests['keyset_sha256']}, expected {expected_keyset}" + ) + if await self.contact_page_mask(page) != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"contact-page corruption target page {page} is not full" + ) + + expected_slots = await self.capture_contact_slot_map( + baseline, "before contact-page corruption" + ) + all_slots = set(range(self.config.contact_count)) + if set(expected_slots.values()) != all_slots: + raise HilFailure( + "full contact inventory is not a bijection over slots 0..349" + ) + slot_order = [ + key for key, _slot in sorted( + expected_slots.items(), key=lambda item: item[1] + ) + ] + if baseline_order != slot_order: + raise HilFailure( + "pre-corruption contact stream is not in persistent slot order" + ) + + first = page * EXPECTED_CONTACTS_PER_PAGE + page_items = sorted( + ( + (slot, key) + for key, slot in expected_slots.items() + if slot // EXPECTED_CONTACTS_PER_PAGE == page + ) + ) + expected_page_slots = list( + range(first, first + EXPECTED_CONTACTS_PER_PAGE) + ) + if [slot for slot, _key in page_items] != expected_page_slots: + raise HilFailure( + f"page {page} did not map to 25 contiguous populated slots" + ) + lost_keys = {key for _slot, key in page_items} + expected_after = { + key: value for key, value in baseline.items() if key not in lost_keys + } + expected_after_order = [ + key for key in baseline_order if key not in lost_keys + ] + + corruption_attempted = False + restored_exactly = False + try: + # The write can succeed even if its reply is lost, so arm emergency + # recovery before issuing the first destructive byte change. + corruption_attempted = True + reply = await self.cli(f"hil extrafs corrupt-page {page} CONFIRM") + expected_reply = f"HIL contact page {page} CRC corrupted" + if reply != expected_reply: + raise HilFailure(f"contact-page corruption failed: {reply!r}") + + await self.reboot_and_reconnect() + after_contacts = await self.contacts() + self.assert_exact_contact_inventory( + after_contacts, + expected_after, + f"after corrupting contact page {page}", + ) + if list(after_contacts) != expected_after_order: + raise HilFailure( + f"surviving stream order changed after corrupting page {page}" + ) + for _slot, key in page_items: + await self.expect_contact_not_found( + key, f"contact from CRC-invalid page {page}" + ) + + restored, restored_digests = ( + await self.restore_contact_inventory_exact( + baseline, + baseline_order, + expected_slots, + page, + current=after_contacts, + context="contact-page CRC restoration", + ) + ) + restored_exactly = True + final_status = await self.status() + if final_status.filler_bytes != 0: + raise HilFailure("contact-page restoration left filler behind") + if final_status.used_kib != initial_status.used_kib: + raise HilFailure( + "contact-page restoration changed ExtraFS usage from " + f"{initial_status.used_kib} KiB to " + f"{final_status.used_kib} KiB" + ) + except BaseException as test_exc: + if corruption_attempted and not restored_exactly: + try: + await self.clear_filler() + # If the corrupt command's reply was lost, this reboot first + # materializes the CRC-invalid page as the expected holes. + # It is also safe after a partial restoration: reboot flushes + # acknowledged records before the resnapshot below. + await self.reboot_and_reconnect() + restored, restored_digests = ( + await self.restore_contact_inventory_exact( + baseline, + baseline_order, + expected_slots, + page, + context="emergency contact-page CRC restoration", + ) + ) + recovery_status = await self.status() + if ( + recovery_status.filler_bytes != 0 + or recovery_status.used_kib != initial_status.used_kib + ): + raise HilFailure( + "emergency contact-page restoration changed " + "ExtraFS usage or left filler behind" + ) + restored_exactly = True + self.record( + "contact_page_corruption_emergency_restore", + page=page, + recovered_exactly=True, + **restored_digests, + ) + except BaseException as restore_exc: + raise HilFailure( + f"contact-page corruption test failed ({test_exc}); " + f"emergency restoration also failed ({restore_exc})" + ) from test_exc + raise + + if restored_digests != baseline_digests: + raise HilFailure( + "contact-page restoration changed exact inventory digests" + ) + self.record( + "contact_page_crc_loss_and_restore", + page=page, + before=len(baseline), + isolated_after=len(expected_after), + isolated_loss=EXPECTED_CONTACTS_PER_PAGE, + lost_slots=expected_page_slots, + lost_key_prefixes=[key[:12] for _slot, key in page_items], + restored=len(restored), + slots_restored=True, + stream_order_restored=True, + baseline_used_kib=initial_status.used_kib, + final_used_kib=final_status.used_kib, + **restored_digests, + ) + return set(baseline) + + async def test_contact_occupied_mask_recovery( + self, page: int, before_contacts: Dict[str, Dict[str, Any]] + ) -> None: + """Corrupt header-only occupancy bits and prove payload-derived repair. + + The populated-slot and invalid-high-bit cases must leave all 350 + contacts untouched. The empty-slot case temporarily removes one + deterministic contact from the selected page, flips that empty slot's + header bit from zero to one, and restores the exact contact before this + method returns. Its emergency path makes the same restoration attempt + after every failure following the acknowledged removal. + """ + + expected_keys = { + make_contact(index)["public_key"] + for index in range(self.config.contact_count) + } + self.assert_exact_contacts( + before_contacts, + expected_keys, + "before occupied-mask corruption", + ) + expected_digest = self.keyset_digest(expected_keys) + initial_mask = await self.contact_page_mask(page) + if initial_mask != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"contact page {page} starts with mask {initial_mask:08x}, " + f"expected {EXPECTED_FULL_PAGE_MASK:08x}" + ) + + cases = ( + (0, EXPECTED_FULL_PAGE_MASK ^ 0x00000001, "valid occupied"), + (31, EXPECTED_FULL_PAGE_MASK | 0x80000000, "invalid high"), + ) + case_results: List[Dict[str, Any]] = [] + current_full_contacts = before_contacts + for bit, corrupted_mask, label in cases: + reply = await self.cli( + f"hil extrafs corrupt-occupied {page} {bit} CONFIRM" + ) + expected_reply = ( + f"HIL contact page {page} occupancy bit {bit} corrupted" + ) + if reply != expected_reply: + raise HilFailure( + f"contact-page occupancy corruption failed: {reply!r}" + ) + observed_corrupt = await self.contact_page_mask(page) + if observed_corrupt != corrupted_mask: + raise HilFailure( + f"{label} bit corruption produced {observed_corrupt:08x}, " + f"expected {corrupted_mask:08x}" + ) + + await self.reboot_and_reconnect() + await self.verify_contacts_after_reboot(expected_keys) + repaired_mask = await self.wait_for_contact_page_mask( + page, EXPECTED_FULL_PAGE_MASK + ) + + # A second reboot proves the repair itself reached ExtraFS, rather + # than merely observing the loader's reconstructed RAM table. + await self.reboot_and_reconnect() + after_repair = await self.contacts() + self.assert_exact_contacts( + after_repair, + expected_keys, + f"after durable {label} mask repair", + ) + actual_digest = self.keyset_digest(after_repair) + if actual_digest != expected_digest: + raise HilFailure( + f"{label} repair keyset digest is {actual_digest}, " + f"expected {expected_digest}" + ) + durable_mask = await self.contact_page_mask(page) + if durable_mask != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"{label} repair did not survive reboot: {durable_mask:08x}" + ) + case_results.append( + { + "kind": label, + "bit": bit, + "corrupted_mask": f"{corrupted_mask:08x}", + "repaired_mask": f"{repaired_mask:08x}", + "durable_mask": f"{durable_mask:08x}", + "contact_count": len(after_repair), + "keyset_sha256": actual_digest, + } + ) + current_full_contacts = after_repair + + # Contact streams are emitted in persistent slot order after reboot. + # Every selected page is full at this point, so the first streamed + # contact for the page owns its valid bit zero. Removing it creates an + # all-zero, CRC-covered payload record whose header bit can safely be + # changed from empty to occupied without inventing a real contact. + page_first = page * EXPECTED_CONTACTS_PER_PAGE + page_keys = list(current_full_contacts)[ + page_first:page_first + EXPECTED_CONTACTS_PER_PAGE + ] + if len(page_keys) != EXPECTED_CONTACTS_PER_PAGE: + raise HilFailure( + f"contact page {page} did not map to " + f"{EXPECTED_CONTACTS_PER_PAGE} populated contacts" + ) + empty_bit = 0 + target_key = page_keys[empty_bit] + if target_key not in expected_keys: + raise HilFailure( + "empty occupancy test target is not a harness-owned " + "deterministic contact" + ) + original_contact = dict(current_full_contacts[target_key]) + original_contact.setdefault("public_key", target_key) + expected_without_target = expected_keys - {target_key} + expected_empty_mask = EXPECTED_FULL_PAGE_MASK & ~(1 << empty_bit) + expected_empty_digest = self.keyset_digest(expected_without_target) + removal_acknowledged = False + original_restored = False + + async def restore_original_contact() -> Dict[str, Dict[str, Any]]: + current = await self.contacts() + if target_key in current: + event = await self.mc.commands.update_contact( + dict(original_contact) + ) + else: + event = await self.mc.commands.add_contact( + dict(original_contact) + ) + self._require_event(event, "OK") + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + restored = await self.contacts() + self.assert_contact_variant( + restored, + expected_keys, + target_key, + original_contact, + "after restoring the empty-mask contact", + ) + restored_digest = self.keyset_digest(restored) + if restored_digest != expected_digest: + raise HilFailure( + f"restored contact keyset digest is {restored_digest}, " + f"expected {expected_digest}" + ) + restored_mask = await self.contact_page_mask(page) + if restored_mask != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + f"restored contact page mask is {restored_mask:08x}, " + f"expected {EXPECTED_FULL_PAGE_MASK:08x}" + ) + return restored + + try: + self._require_event( + await self.mc.commands.remove_contact(target_key), "OK" + ) + removal_acknowledged = True + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + after_removal = await self.contacts() + self.assert_exact_contacts( + after_removal, + expected_without_target, + "after removing the empty-mask test contact", + ) + removed_mask = await self.contact_page_mask(page) + if removed_mask != expected_empty_mask: + raise HilFailure( + f"contact removal produced page mask {removed_mask:08x}, " + f"expected {expected_empty_mask:08x}" + ) + + reply = await self.cli( + f"hil extrafs corrupt-occupied {page} {empty_bit} CONFIRM" + ) + expected_reply = ( + f"HIL contact page {page} occupancy bit {empty_bit} corrupted" + ) + if reply != expected_reply: + raise HilFailure( + f"empty contact-page occupancy corruption failed: {reply!r}" + ) + observed_corrupt = await self.contact_page_mask(page) + if observed_corrupt != EXPECTED_FULL_PAGE_MASK: + raise HilFailure( + "empty-slot bit corruption produced " + f"{observed_corrupt:08x}, expected " + f"{EXPECTED_FULL_PAGE_MASK:08x}" + ) + + await self.reboot_and_reconnect() + after_empty_repair = await self.contacts() + self.assert_exact_contacts( + after_empty_repair, + expected_without_target, + "after loading an empty record marked occupied", + ) + repaired_mask = await self.wait_for_contact_page_mask( + page, expected_empty_mask + ) + + # Reboot once more before restoring the contact so this test proves + # that the corrected empty mask, not just the reconstructed RAM + # table, was committed durably. + await self.reboot_and_reconnect() + durable_empty = await self.contacts() + self.assert_exact_contacts( + durable_empty, + expected_without_target, + "after durable empty-slot mask repair", + ) + durable_mask = await self.contact_page_mask(page) + if durable_mask != expected_empty_mask: + raise HilFailure( + f"empty-slot repair did not survive reboot: " + f"{durable_mask:08x}" + ) + durable_digest = self.keyset_digest(durable_empty) + if durable_digest != expected_empty_digest: + raise HilFailure( + f"empty-slot repair keyset digest is {durable_digest}, " + f"expected {expected_empty_digest}" + ) + + restored = await restore_original_contact() + original_restored = True + case_results.append( + { + "kind": "valid empty", + "bit": empty_bit, + "removed_key_prefix": target_key[:12], + "before_corruption_mask": f"{expected_empty_mask:08x}", + "corrupted_mask": f"{EXPECTED_FULL_PAGE_MASK:08x}", + "repaired_mask": f"{repaired_mask:08x}", + "durable_mask": f"{durable_mask:08x}", + "contact_count": len(durable_empty), + "keyset_sha256": durable_digest, + "restored_contact_count": len(restored), + "restored_mask": f"{EXPECTED_FULL_PAGE_MASK:08x}", + } + ) + except BaseException as test_exc: + if removal_acknowledged and not original_restored: + try: + await restore_original_contact() + original_restored = True + self.record( + "contact_occupied_mask_emergency_restore", + page=page, + bit=empty_bit, + target_key_prefix=target_key[:12], + original_restored=True, + ) + except BaseException as restore_exc: + raise HilFailure( + f"empty occupancy test failed ({test_exc}); emergency " + f"contact restore also failed ({restore_exc})" + ) from test_exc + raise + + self.record( + "contact_occupied_mask_recovery", + page=page, + expected_mask=f"{EXPECTED_FULL_PAGE_MASK:08x}", + contacts_preserved=len(expected_keys), + keyset_sha256=expected_digest, + original_contact_restored=original_restored, + cases=case_results, + ) + + async def cleanup_generated_contacts(self) -> None: + await self.clear_filler() + present = await self.contacts() + synthetic_keys = { + make_contact(index)["public_key"] + for index in range(self.config.contact_count) + } + expected_remaining = set(present) - synthetic_keys + if self.baseline_contact_keys: + expected_remaining = set(self.baseline_contact_keys) + removed = 0 + for public_key in sorted(set(present) & synthetic_keys): + if public_key not in present: + continue + self._require_event( + await self.mc.commands.remove_contact(public_key), "OK" + ) + removed += 1 + if removed % 25 == 0: + self.progress(f"contacts removed: {removed}") + await asyncio.sleep(self.config.settle_seconds) + await self.reboot_and_reconnect() + remaining_contacts = await self.contacts() + self.assert_exact_contacts( + remaining_contacts, + expected_remaining, + "after synthetic-contact cleanup", + ) + self.record( + "cleanup", + removed=removed, + remaining=len(remaining_contacts), + keyset_sha256=self.keyset_digest(remaining_contacts), + filler_cleared=True, + ) + + async def restore_test_channel(self) -> None: + if self.original_channel_snapshot is None: + return + await self.clear_filler() + expected = self.original_channel_snapshot + current = self._require_event( + await self.mc.commands.get_channel(self.config.channel_index), + "CHANNEL_INFO", + ).payload + if ( + current.get("channel_name") != expected["channel_name"] + or current.get("channel_secret") != expected["channel_secret"] + ): + self._require_event( + await self.set_channel_exact( + self.config.channel_index, + expected["channel_name"], + expected["channel_secret"], + ), + "OK", + ) + self.record( + "test_channel_restore_staged", + channel_index=self.config.channel_index, + ) + + async def comprehensive(self) -> None: + # Recover cleanly from an earlier interrupted ENOSPC run before adding + # contacts. This is destructive and therefore occurs only after both + # command-line consent gates have passed. + await self.clear_filler() + await self.status() + self_info = self._require_event( + await self.mc.commands.send_appstart(), "SELF_INFO" + ).payload + autoadd = self._require_event( + await self.mc.commands.get_autoadd_config(), "AUTOADD_CONFIG" + ).payload + self.original_manual_add_contacts = bool( + self_info.get("manual_add_contacts", False) + ) + self.original_autoadd_config = int(autoadd.get("config", 0)) + try: + if not self.original_manual_add_contacts: + await self.set_manual_contact_mode(True) + if self.original_autoadd_config != 0: + await self.set_autoadd_config(0) + expected_contacts = await self.add_contacts_to_capacity() + + # A RAM count is not persistence proof. Reboot while there is + # still free space before creating the ENOSPC condition. + await self.reboot_and_reconnect() + await self.verify_contacts_after_reboot(expected_contacts) + + if self.config.unsafe_reset: + await self.reboot_and_reconnect(unsafe=True) + await self.verify_contacts_after_reboot(expected_contacts) + + fill, original_channel = await self.fill_and_test_channel_atomicity() + await self.reboot_and_reconnect() + await self.verify_contacts_after_reboot(expected_contacts) + await self.verify_full_state_after_reboot(fill, original_channel) + + await self.test_contact_update_recovery_when_full(expected_contacts) + await self.test_contact_add_recovery_when_full(expected_contacts) + await self.test_contact_remove_recovery_when_full(expected_contacts) + if getattr(self.config, "advert_remove_rollback", False): + await self.test_advert_backed_remove_rollback(expected_contacts) + if getattr(self.config, "fail_read_page", None) is not None: + await self.test_contact_page_read_failure( + self.config.fail_read_page + ) + if getattr(self.config, "fail_stat_page", None) is not None: + await self.test_contact_page_stat_failure( + self.config.fail_stat_page + ) + + await self.test_channel_persistence_when_free() + if self.config.corrupt_occupied_page is not None: + await self.test_contact_occupied_mask_recovery( + self.config.corrupt_occupied_page, await self.contacts() + ) + if self.config.corrupt_page is not None: + before_corruption = await self.contacts() + expected_contacts = await self.corrupt_contact_page( + self.config.corrupt_page, before_corruption + ) + self.expected_contact_keys = expected_contacts + await self.status() + finally: + active_failure = sys.exc_info()[0] is not None + restore_errors: List[str] = [] + + if self.mc is not None and self.config.cleanup: + try: + await self.restore_test_channel() + except BaseException as restore_exc: + restore_errors.append(f"channel/filler: {restore_exc}") + try: + await self.cleanup_generated_contacts() + except BaseException as restore_exc: + restore_errors.append(f"contacts: {restore_exc}") + + if self.mc is not None and self.original_autoadd_config is not None: + try: + current = self._require_event( + await self.mc.commands.get_autoadd_config(), + "AUTOADD_CONFIG", + ).payload + if int(current.get("config", -1)) != self.original_autoadd_config: + await self.set_autoadd_config(self.original_autoadd_config) + except BaseException as restore_exc: + restore_errors.append(f"autoadd config: {restore_exc}") + + if self.mc is not None and self.original_manual_add_contacts is not None: + try: + current = self._require_event( + await self.mc.commands.send_appstart(), "SELF_INFO" + ).payload + if bool(current.get("manual_add_contacts")) != ( + self.original_manual_add_contacts + ): + await self.set_manual_contact_mode( + self.original_manual_add_contacts + ) + except BaseException as restore_exc: + restore_errors.append(f"manual contact mode: {restore_exc}") + + if restore_errors: + self.record( + "contact_isolation_restore_failed", + errors=restore_errors, + ) + if not active_failure: + raise HilFailure("; ".join(restore_errors)) + + +async def run(config: RunConfig) -> Dict[str, Any]: + result: Dict[str, Any] = { + "schema_version": 1, + "tool": "t1000e_extrafs_stress", + "ok": False, + "config": _jsonable(asdict(config)), + "locked_identity": { + "usb_serial": EXPECTED_USB_SERIAL, + "expected_node_key_prefix": config.expected_node_key_prefix, + "known_pre_erase_node_key_prefix": KNOWN_PRE_ERASE_NODE_KEY_PREFIX, + "vid_pid": f"{EXPECTED_USB_VID:04X}:{EXPECTED_USB_PID:04X}", + "interface": EXPECTED_USB_INTERFACE, + }, + "steps": [], + "failure": None, + } + runner: Optional[StressRunner] = None + try: + require_destructive_consent(config) + identity = resolve_initial_port(config.port) + runner = StressRunner(config, identity) + await runner.connect() + if config.scenario == "inspect": + await runner.inspect() + else: + await runner.comprehensive() + result["ok"] = True + except BaseException as exc: + result["failure"] = { + "type": type(exc).__name__, + "message": str(exc), + } + finally: + if runner is not None: + result["steps"] = runner.steps + try: + await runner.disconnect() + except BaseException as exc: + if result["failure"] is None: + result["failure"] = { + "type": type(exc).__name__, + "message": str(exc), + } + result["ok"] = False + return result + + +def _positive_int(value: str) -> int: + parsed = int(value, 10) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + +def _nonnegative_page(value: str) -> int: + parsed = int(value, 10) + if not 0 <= parsed <= 13: + raise argparse.ArgumentTypeError("must be in 0..13") + return parsed + + +def _positive_float(value: str) -> float: + parsed = float(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", help="initial Companion port (for example COM23)") + parser.add_argument( + "--scenario", + choices=("inspect", "comprehensive"), + default="inspect", + help="inspect is read-only and is the default", + ) + parser.add_argument("--allow-destructive", action="store_true") + parser.add_argument( + "--confirm-usb-serial", + help="must exactly name the locked spare for any destructive plan", + ) + parser.add_argument( + "--expected-node-key-prefix", + help=( + "optional data-identity check; omit after an intentional erase " + "because erasing regenerates the node key" + ), + ) + parser.add_argument("--contact-count", type=_positive_int, default=350) + parser.add_argument("--fill-bytes", type=_positive_int, default=131072) + parser.add_argument("--channel-index", type=int, default=39) + parser.add_argument("--settle-seconds", type=_positive_float, default=7.0) + parser.add_argument("--reenum-timeout", type=_positive_float, default=35.0) + parser.add_argument("--allow-existing-contacts", action="store_true") + parser.add_argument( + "--contact-enumeration-passes", + type=_positive_int, + metavar="N", + help=( + "run N validated full, unfiltered contact-list passes at each " + "inventory/reboot checkpoint; retries are counted and make the " + "stress result fail" + ), + ) + parser.add_argument( + "--expected-contact-keyset-sha256", + metavar="HEX64", + help=( + "optional expected digest of sorted contact public keys; useful " + "for anchoring a read-only enumeration run" + ), + ) + parser.add_argument("--cleanup", action="store_true") + parser.add_argument( + "--corrupt-page", + type=_nonnegative_page, + metavar="0..13", + help=( + "corrupt one persisted contact page, verify isolated loss, then " + "restore its exact records, slots, and stream order" + ), + ) + parser.add_argument( + "--corrupt-occupied-page", + type=_nonnegative_page, + metavar="0..13", + help=( + "also flip populated, empty, and invalid occupied-mask bits, then " + "verify payload-derived recovery, durable header repair, and exact " + "contact restoration; any existing contacts must be the harness's " + "deterministic set" + ), + ) + parser.add_argument( + "--unsafe-reset", + action="store_true", + help=( + "exercise the persistence-bypassing reset while USB is awake; " + "this is not an event-driven sleep test" + ), + ) + parser.add_argument( + "--advert-remove-rollback", + action="store_true", + help=( + "seed a real cached advert and prove its ENOSPC removal rolls " + "back the live contact and persistent slot exactly; requires the " + "deterministic 350-contact inventory" + ), + ) + parser.add_argument( + "--fail-read-page", + type=_nonnegative_page, + metavar="0..13", + help=( + "inject one non-destructive contact-page read failure on the next " + "boot and prove fail-closed enumeration/mutations plus exact " + "recovery on the following boot" + ), + ) + parser.add_argument( + "--fail-stat-page", + type=_nonnegative_page, + metavar="0..13", + help=( + "inject one non-destructive contact-page presence-stat failure on " + "the next boot and prove source discovery/mutations fail closed " + "plus exact recovery on the following boot" + ), + ) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--pretty", action="store_true") + parser.add_argument( + "--self-test", + action="store_true", + help="run offline parser/safety tests; never enumerate or open hardware", + ) + return parser + + +def config_from_args(args: argparse.Namespace) -> RunConfig: + # getattr keeps config construction compatible with callers that build a + # Namespace rather than obtaining it from build_argument_parser(). + corrupt_occupied_page = getattr(args, "corrupt_occupied_page", None) + advert_remove_rollback = bool( + getattr(args, "advert_remove_rollback", False) + ) + fail_read_page = getattr(args, "fail_read_page", None) + fail_stat_page = getattr(args, "fail_stat_page", None) + if not args.port: + raise HilFailure("--port is required unless --self-test is used") + if not 0 <= args.channel_index <= 39: + raise HilFailure("--channel-index must be in 0..39") + if args.contact_count != 350: + raise HilFailure( + "this T1000-E capacity harness requires --contact-count 350" + ) + if args.expected_node_key_prefix: + prefix = args.expected_node_key_prefix + if len(prefix) < 8 or len(prefix) > 64 or len(prefix) % 2: + raise HilFailure( + "--expected-node-key-prefix must be 8..64 hexadecimal " + "characters with even length" + ) + try: + bytes.fromhex(prefix) + except ValueError as exc: + raise HilFailure( + "--expected-node-key-prefix must be hexadecimal" + ) from exc + if args.fill_bytes > 131072: + raise HilFailure("--fill-bytes must not exceed the firmware HIL limit 131072") + if fail_read_page is not None and not 0 <= fail_read_page <= 13: + raise HilFailure("--fail-read-page must be in 0..13") + if fail_stat_page is not None and not 0 <= fail_stat_page <= 13: + raise HilFailure("--fail-stat-page must be in 0..13") + if fail_read_page is not None and fail_stat_page is not None: + raise HilFailure( + "--fail-read-page and --fail-stat-page are mutually exclusive" + ) + expected_keyset_digest = None + if args.expected_contact_keyset_sha256 is not None: + expected_keyset_digest = args.expected_contact_keyset_sha256.lower() + if args.contact_enumeration_passes is None: + raise HilFailure( + "--expected-contact-keyset-sha256 requires " + "--contact-enumeration-passes" + ) + if len(expected_keyset_digest) != 64: + raise HilFailure( + "--expected-contact-keyset-sha256 must be 64 hexadecimal characters" + ) + try: + bytes.fromhex(expected_keyset_digest) + except ValueError as exc: + raise HilFailure( + "--expected-contact-keyset-sha256 must be hexadecimal" + ) from exc + if ( + args.corrupt_page is not None + or corrupt_occupied_page is not None + or advert_remove_rollback + or fail_read_page is not None + or fail_stat_page is not None + or args.unsafe_reset + ) and ( + args.scenario != "comprehensive" + ): + raise HilFailure( + "corruption, read-failure, advert rollback, and unsafe-reset " + "options require --scenario comprehensive" + ) + if args.cleanup and args.scenario != "comprehensive": + raise HilFailure("--cleanup requires --scenario comprehensive") + return RunConfig( + port=args.port, + scenario=args.scenario, + allow_destructive=args.allow_destructive, + confirm_usb_serial=args.confirm_usb_serial, + expected_node_key_prefix=args.expected_node_key_prefix, + contact_count=args.contact_count, + fill_bytes=args.fill_bytes, + channel_index=args.channel_index, + settle_seconds=args.settle_seconds, + reenum_timeout=args.reenum_timeout, + allow_existing_contacts=args.allow_existing_contacts, + cleanup=args.cleanup, + corrupt_page=args.corrupt_page, + corrupt_occupied_page=corrupt_occupied_page, + unsafe_reset=args.unsafe_reset, + contact_enumeration_passes=args.contact_enumeration_passes, + expected_contact_keyset_sha256=expected_keyset_digest, + verbose=args.verbose, + advert_remove_rollback=advert_remove_rollback, + fail_read_page=fail_read_page, + fail_stat_page=fail_stat_page, + ) + + +def offline_self_test() -> Dict[str, Any]: + """Exercise parsers and safety gates without importing serial libraries.""" + + checks: List[str] = [] + status = parse_extrafs_status( + "HIL ExtraFS active=1 used=73KiB total=100KiB fill=12345" + ) + assert status == ExtraFsStatus(True, 73, 100, 12345) + checks.append("status_parser") + + fill = parse_extrafs_fill( + "HIL ExtraFS fill requested=131072 written=20480 " + "used=100KiB total=100KiB" + ) + assert fill == ExtraFsFill(131072, 20480, 100, 100) + checks.append("fill_parser") + + page_mask = parse_contact_page_mask( + "HIL contact page 13 occupied=81fF00a5" + ) + assert page_mask == ContactPageMask(13, 0x81FF00A5) + checks.append("contact_page_mask_parser") + + assert parse_contact_slot("HIL contact slot=349") == 349 + assert parse_advert_seed_slot("HIL advert seeded slot=12") == 12 + checks.append("advert_hil_reply_parsers") + + for malformed in ( + "HIL ExtraFS active=1 used=101KiB total=100KiB fill=0", + "not a HIL response", + ): + try: + parse_extrafs_status(malformed) + except HilFailure: + pass + else: + raise AssertionError(f"status parser accepted {malformed!r}") + checks.append("malformed_status_rejected") + + first = make_contact(0) + second = make_contact(1) + assert len(first["public_key"]) == 64 + assert first["public_key"] != second["public_key"] + checks.append("deterministic_contacts") + + first["lastmod"] = 1_800_000_000 + exact_frame = encode_exact_contact_frame(first) + assert len(exact_frame) == 148 + assert int.from_bytes(exact_frame[-4:], "little") == first["lastmod"] + checks.append("exact_contact_frame") + + safe = RunConfig( + "COM23", "inspect", False, None, None, 350, 131072, 39, 7, 35, + False, False, None, False, None, None, False, + ) + require_destructive_consent(safe) + denied = RunConfig( + "COM23", "comprehensive", False, None, None, 350, 131072, 39, 7, + 35, False, False, None, False, None, None, False, + ) + try: + require_destructive_consent(denied) + except HilFailure: + pass + else: + raise AssertionError("destructive plan passed without consent") + allowed = RunConfig( + "COM23", "comprehensive", True, EXPECTED_USB_SERIAL, None, 350, + 131072, 39, 7, 35, False, False, None, False, None, None, False, + ) + require_destructive_consent(allowed) + checks.append("destructive_consent_gate") + + identity = PortIdentity( + "COM23", EXPECTED_USB_SERIAL, EXPECTED_USB_VID, EXPECTED_USB_PID, + "MeshCore T1000-E", "MeshCore T1000-E", "", "1-8:x.0", + f"USB VID:PID={EXPECTED_USB_VID:04X}:{EXPECTED_USB_PID:04X}", + ) + validate_runtime_identity(identity, requested_port="com23") + wrong = PortIdentity( + "COM23", "WRONG", EXPECTED_USB_VID, EXPECTED_USB_PID, + "MeshCore T1000-E", "", "", "1-8", "USB MI_00", + ) + try: + validate_runtime_identity(wrong, requested_port="COM23") + except HilFailure: + pass + else: + raise AssertionError("wrong USB serial passed the identity gate") + checks.append("usb_identity_gate") + + parser = build_argument_parser() + parsed = parser.parse_args(["--port", "COM23"]) + assert ( + parsed.scenario == "inspect" + and not parsed.allow_destructive + and not parsed.advert_remove_rollback + and parsed.fail_read_page is None + and parsed.fail_stat_page is None + and parsed.contact_enumeration_passes is None + ) + checks.append("read_only_default") + + stress_args = parser.parse_args( + [ + "--port", + "COM23", + "--contact-enumeration-passes", + "5", + "--expected-contact-keyset-sha256", + "ab" * 32, + ] + ) + stress_config = config_from_args(stress_args) + assert stress_config.contact_enumeration_passes == 5 + assert stress_config.expected_contact_keyset_sha256 == "ab" * 32 + require_destructive_consent(stress_config) + checks.append("read_only_enumeration_config") + + read_failure_args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--fail-read-page", + "13", + ] + ) + read_failure_config = config_from_args(read_failure_args) + assert read_failure_config.fail_read_page == 13 + checks.append("read_failure_config") + stat_failure_args = parser.parse_args( + [ + "--port", + "COM23", + "--scenario", + "comprehensive", + "--fail-stat-page", + "13", + ] + ) + stat_failure_config = config_from_args(stat_failure_args) + assert stat_failure_config.fail_stat_page == 13 + checks.append("stat_failure_config") + return {"ok": True, "checks": checks} + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_argument_parser() + args = parser.parse_args(argv) + if args.self_test: + result = offline_self_test() + print(json.dumps(result, indent=2 if args.pretty else None, sort_keys=True)) + return 0 + try: + config = config_from_args(args) + result = asyncio.run(run(config)) + except BaseException as exc: + result = { + "schema_version": 1, + "tool": "t1000e_extrafs_stress", + "ok": False, + "steps": [], + "failure": {"type": type(exc).__name__, "message": str(exc)}, + } + print(json.dumps(result, indent=2 if args.pretty else None, sort_keys=True)) + return 0 if result["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main())