diff --git a/examples/companion_radio/DataStore.cpp b/examples/companion_radio/DataStore.cpp index c895a699c..ee304993c 100644 --- a/examples/companion_radio/DataStore.cpp +++ b/examples/companion_radio/DataStore.cpp @@ -2531,7 +2531,7 @@ bool DataStore::serviceContactWrite(DataStoreHost* host, // The revision/session guard above also applies to every setup step. // Keep initialization separate from serialization, including its final // buffer configuration, so transports are serviced between operations. - const auto progress = _contact_write->serviceBegin(); + const auto progress = _contact_write->serviceBegin(true); if (progress == mesh::ContactFileTransaction::BeginProgress::Failed) { cancelContactWrite(); return false; @@ -2581,7 +2581,7 @@ bool DataStore::serviceContactWrite(DataStoreHost* host, #endif // Keep the 64-byte CRC scratch while verifying at most 512 bytes per // background pass; transports run between these bounded batches. - const auto progress = _contact_write->serviceCommit(true, 8); + const auto progress = _contact_write->serviceCommit(true, 8, true); if (progress == mesh::ContactFileTransaction::CommitProgress::Pending) { return true; } diff --git a/src/helpers/ContactFileTransaction.h b/src/helpers/ContactFileTransaction.h index b9c0bc3ec..2bd088985 100644 --- a/src/helpers/ContactFileTransaction.h +++ b/src/helpers/ContactFileTransaction.h @@ -33,11 +33,12 @@ private: // This bounded stdio buffer coalesces records without the SDK's 4 KiB burst; // one logical record can flush at most one full buffer per write pass. static constexpr size_t ESP_WRITE_BUFFER_SIZE = 251; + static constexpr size_t ESP_BACKGROUND_WRITE_BUFFER_SIZE = 502; static constexpr size_t ESP_VERIFY_BUFFER_SIZE = 128; static_assert(storage::CONTACT_RECORD_SIZE <= ESP_WRITE_BUFFER_SIZE, "Contact records must fit one bounded writer buffer"); enum class BeginStage : uint8_t { - Target, Backup, Recover, Temp, Remove, Open, Configure, Ready, Failed + Target, Backup, Recover, RetireBackup, Temp, Remove, Open, Configure, Ready, Failed }; BeginStage _begin_stage = BeginStage::Target; bool _begin_target_exists = false; @@ -106,7 +107,7 @@ public: operator bool() const { return _ok; } #if defined(ESP32_PLATFORM) enum class BeginProgress : uint8_t { Pending, Ready, Failed }; - BeginProgress serviceBegin() { + BeginProgress serviceBegin(bool background_contact_write = false) { bool ok = true; switch (_begin_stage) { case BeginStage::Target: @@ -118,8 +119,22 @@ public: if (ok) _begin_stage = BeginStage::Recover; break; case BeginStage::Recover: - if (!_begin_target_exists && _begin_backup_exists) + if (!_begin_target_exists && _begin_backup_exists) { ok = _fs->rename(_backup, _target); + if (ok) { + _begin_target_exists = true; + _begin_backup_exists = false; + } + } + if (ok) _begin_stage = background_contact_write + ? BeginStage::RetireBackup : BeginStage::Temp; + break; + case BeginStage::RetireBackup: + // A successful prior publication may retain its old backup. Retire it + // in a separate setup pass only when the original target was present. + // Recover already consumed the backup when the target was missing. + if (_begin_target_exists && _begin_backup_exists) + ok = _fs->remove(_backup); if (ok) _begin_stage = BeginStage::Temp; break; case BeginStage::Temp: @@ -140,9 +155,11 @@ public: if (ok) _begin_stage = BeginStage::Configure; break; case BeginStage::Configure: - // Arduino otherwise batches small writes into a 4 KiB stdio buffer. - // Configure the newly opened stream before its first I/O. - ok = _file.setBufferSize(ESP_WRITE_BUFFER_SIZE); + // Select once at Configure, before the stream's first I/O. Default + // and synchronous callers retain one 251-byte SPIFFS payload page; + // background contacts coalesce two pages without a larger record. + ok = _file.setBufferSize(background_contact_write + ? ESP_BACKGROUND_WRITE_BUFFER_SIZE : ESP_WRITE_BUFFER_SIZE); if (ok) { _ok = true; _begin_stage = BeginStage::Ready; @@ -176,7 +193,8 @@ public: // callers may request up to eight chunks while reusing the same scratch // buffer. Transports run between passes without exposing the rename gap; // synchronous commit() and other platforms retain the one-chunk default. - CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) { + CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1, + bool defer_backup_cleanup = false) { #if defined(ESP32_PLATFORM) // A premature commit must not publish, clean up, or implicitly aggregate // the deferred setup operations inside this call. @@ -193,11 +211,13 @@ public: _verify = _fs->open(_temp, "r"); ok = static_cast(_verify); #if defined(ESP32_PLATFORM) - // fread() otherwise reads ahead up to 4 KiB while the caller requests - // only one 64-byte CRC chunk. A bounded two-chunk read-ahead buffer - // halves backend reads without growing that scratch buffer. The caller's - // separate chunk budget bounds each pass. Configuration must succeed. - ok = ok && _verify.setBufferSize(ESP_VERIFY_BUFFER_SIZE); + // Select read-ahead at Open only. Default/synchronous callers retain + // 128 bytes; an eight-chunk background pass may coalesce its 512-byte + // budget in one backend refill. Later budget changes retain this choice, + // with at most 512 backend bytes per pass and the same 64-byte scratch. + // Configuration must succeed before verification or publication. + ok = ok && _verify.setBufferSize( + max_verify_chunks >= 8 ? 512 : ESP_VERIFY_BUFFER_SIZE); #endif ok = ok && _verify.size() == _size; if (ok) { @@ -252,8 +272,16 @@ public: if (!ok && backed_up) _fs->rename(_backup, _target); // Cleanup remains best effort after the publish/rollback result, but an // absent file must not trigger the SDK's error log on the shared USB port. - if (ok) removeIfPresent(_fs, _backup, _presence); - else removeIfPresent(_fs, _temp, _presence); + if (ok) { +#if defined(ESP32_PLATFORM) + // Background contacts publish the target and its path handles now. + // A later transaction retires this recoverable old backup separately. + if (!defer_backup_cleanup) +#else + (void)defer_backup_cleanup; // Other platforms retain immediate cleanup. +#endif + removeIfPresent(_fs, _backup, _presence); + } else removeIfPresent(_fs, _temp, _presence); _finished = true; return ok ? CommitProgress::Succeeded : CommitProgress::Failed; } diff --git a/test/fixtures/contact_cache/test_contact_cache.cpp b/test/fixtures/contact_cache/test_contact_cache.cpp index 9c69e0dbf..2b2c0fab4 100644 --- a/test/fixtures/contact_cache/test_contact_cache.cpp +++ b/test/fixtures/contact_cache/test_contact_cache.cpp @@ -489,11 +489,11 @@ static void background_crc_verification_uses_bounded_batches() { total_crc_reads += read_count; max_reads = std::max(max_reads, read_count); max_backend_reads = std::max(max_backend_reads, backend_count); - assert(read_count <= 8 && backend_count <= 4); + assert(read_count <= 8 && backend_count <= 1); } assert(total_crc_reads == 825); assert(crc_read_passes == 104); - assert(max_reads == 8 && max_backend_reads == 4); + assert(max_reads == 8 && max_backend_reads == 1); assert(!SPIFFS.exists("/contacts3.tmp")); f.host.contacts.clear(); f.store.loadContacts(&f.host); @@ -818,9 +818,30 @@ static void contact_tx_policy_round_trips_without_growing_records() { } } + +#if defined(ESP32_PLATFORM) +static void deferred_cleanup_publishes_actual_cached_handles() { + Fixture f(40); + const auto old = SPIFFS.files.at("/contacts3"); + SPIFFS.files["/contacts3.bak"] = {9, 8, 7}; + assert(f.host.contacts[0].setRawPath(route(2100).data())); + assert(f.store.markContactDirty(f.host.contacts[0])); + for (unsigned step = 0; step < 4; ++step) + assert(f.store.serviceContactWrites(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3.bak") == std::vector({9, 8, 7})); + assert(f.store.serviceContactWrites(&f.host, cachedContactFilter)); + assert(!SPIFFS.exists("/contacts3.bak") && SPIFFS.files.at("/contacts3") == old); + drain_cooperative_save(f); + assert(!f.store.hasPendingContactWrites() && SPIFFS.files.at("/contacts3.bak") == old); + f.check(0, route(2100)); + for (unsigned i = 1; i < 40; ++i) f.check(i, route(i)); +} +#endif + int main() { contact_tx_policy_round_trips_without_growing_records(); #if defined(ESP32_PLATFORM) + deferred_cleanup_publishes_actual_cached_handles(); cooperative_save_services_commands_and_preserves_snapshots(); background_crc_verification_uses_bounded_batches(); cooperative_mutations_never_publish_obsolete_indices(); diff --git a/test/fixtures/contact_crc_batch/test.cpp b/test/fixtures/contact_crc_batch/test.cpp index 056025fc9..e953da7fc 100644 --- a/test/fixtures/contact_crc_batch/test.cpp +++ b/test/fixtures/contact_crc_batch/test.cpp @@ -41,6 +41,7 @@ public: unsigned config_calls = 0, closes = 0, renames = 0; unsigned fail_config = 0, short_read_call = 0, fail_rename = 0; size_t largest_read = 0, largest_backend_read = 0, max_write = SIZE_MAX; + size_t backend_read_bytes = 0, reader_buffer_size = 0; bool fail_open_read = false, fail_open_write = false; std::vector rename_snapshots; bool exists(const char* path) { return files.count(path) != 0; } @@ -65,6 +66,7 @@ bool File::setBufferSize(size_t size) { if (++stream->fs->config_calls == stream->fs->fail_config) return false; assert(stream->buffer.empty() && stream->position == 0); stream->buffer_size = size; + if (!stream->writing) stream->fs->reader_buffer_size = size; return true; } size_t File::size() const { return stream ? stream->fs->files.at(stream->path).size() : 0; } @@ -104,6 +106,7 @@ size_t File::read(uint8_t* data, size_t length) { const size_t n = std::min(stream->buffer_size, available); if (!n) break; ++fs.backend_reads; + fs.backend_read_bytes += n; fs.largest_backend_read = std::max(fs.largest_backend_read, n); stream->buffer.assign(source.begin() + stream->backend_position, source.begin() + stream->backend_position + n); @@ -147,6 +150,7 @@ unsigned effectiveBudget(unsigned requested) { struct Measurement { unsigned passes, logical_reads, backend_reads, max_pass_reads, max_pass_refills; + size_t max_pass_backend_bytes, reader_buffer_size; }; Measurement verify(size_t bytes, unsigned budget, bool use_default = false) { FileSystem fs; @@ -156,19 +160,28 @@ Measurement verify(size_t bytes, unsigned budget, bool use_default = false) { assert(writer); writeImage(writer, next); unsigned passes = 0, max_reads = 0, max_refills = 0, data_passes = 0; + size_t max_backend_bytes = 0; +#if defined(ESP32_PLATFORM) + // Selection is made once in Open, independently of the later read loop. + const size_t selected = !use_default && budget >= 8 ? 512 : 128; +#endif Progress result = Progress::Pending; while (result == Progress::Pending) { const unsigned before_reads = fs.reads, before_backend = fs.backend_reads; + const size_t before_backend_bytes = fs.backend_read_bytes; const bool was_ready = writer.readyToPublish(); result = use_default ? writer.serviceCommit() : writer.serviceCommit(true, budget); assert(++passes < 10000); const unsigned reads = fs.reads - before_reads, refills = fs.backend_reads - before_backend; max_reads = std::max(max_reads, reads); max_refills = std::max(max_refills, refills); + const size_t backend_bytes = fs.backend_read_bytes - before_backend_bytes; + max_backend_bytes = std::max(max_backend_bytes, backend_bytes); const unsigned bound = use_default ? 1 : effectiveBudget(budget); assert(reads <= bound); #if defined(ESP32_PLATFORM) - assert(refills <= (bound + 1) / 2); + assert(refills <= (64 * bound + selected - 1) / selected); + assert(backend_bytes <= 512); // FILE refill admissions, not NOR operations or time #endif if (reads) { ++data_passes; @@ -185,10 +198,12 @@ Measurement verify(size_t bytes, unsigned budget, bool use_default = false) { assert(fs.reads == (bytes + 63) / 64); assert(fs.largest_read <= 64); #if defined(ESP32_PLATFORM) - assert(fs.largest_backend_read <= 128); - assert(fs.backend_reads == (bytes + 127) / 128); + assert(fs.reader_buffer_size == selected); + assert(fs.largest_backend_read <= selected); + assert(fs.backend_reads == (bytes + selected - 1) / selected); #endif - return {passes, fs.reads, fs.backend_reads, max_reads, max_refills}; + return {passes, fs.reads, fs.backend_reads, max_reads, max_refills, + max_backend_bytes, fs.reader_buffer_size}; } void limitsAndTails() { for (size_t bytes : {size_t(0), size_t(1), size_t(63), size_t(64), size_t(65), @@ -203,6 +218,9 @@ void limitsAndTails() { fixture_commit_calls = 0; assert(writer.commit()); assert(fixture_commit_calls == 829); // actual commit() drains default-one passes +#if defined(ESP32_PLATFORM) + assert(fs.reader_buffer_size == 128 && fs.backend_reads == 413); +#endif } void readFailureAndCorruption() { const auto old = image(19), next = image(1536); @@ -300,7 +318,11 @@ int main() { #endif const auto one = verify(347 * 152, 1), eight = verify(347 * 152, 8); printf("{\"one_passes\":%u,\"eight_passes\":%u,\"logical_reads\":%u," - "\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u}\n", + "\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u," + "\"max_pass_backend_bytes\":%zu,\"reader_buffer_size\":%zu," + "\"one_backend_reads\":%u,\"one_reader_buffer_size\":%zu}\n", one.passes, eight.passes, eight.logical_reads, eight.backend_reads, - eight.max_pass_reads, eight.max_pass_refills); + eight.max_pass_reads, eight.max_pass_refills, + eight.max_pass_backend_bytes, eight.reader_buffer_size, + one.backend_reads, one.reader_buffer_size); } diff --git a/test/fixtures/contact_persistence/test_esp32_uncached.cpp b/test/fixtures/contact_persistence/test_esp32_uncached.cpp index bf31bca39..3c5a25fb6 100644 --- a/test/fixtures/contact_persistence/test_esp32_uncached.cpp +++ b/test/fixtures/contact_persistence/test_esp32_uncached.cpp @@ -137,7 +137,7 @@ unsigned drain(DataStore& store, Host& h, Filter filter = nullptr) { assert(SPIFFS.writes - writes <= 1); assert(SPIFFS.reads - reads <= 8); assert(SPIFFS.backend_writes - backend_writes <= 1); - assert(SPIFFS.backend_reads - backend_reads <= 4); + assert(SPIFFS.backend_reads - backend_reads <= 1); // Transport service runs here, between every production pass. It must // never observe an unpublished replacement or a target-name gap. if (store.hasPendingContactWrites()) { @@ -171,8 +171,8 @@ void boundedFullTableAndFirstSave() { const auto passes = drain(store, h, keepContact); assert(passes > h.contacts.size()); // serialization and CRC both yield assert(SPIFFS.largest_read <= 64); - assert(SPIFFS.largest_backend_read <= 128); - assert(SPIFFS.largest_backend_write <= 251); + assert(SPIFFS.largest_backend_read <= 512); // background selected at Open + assert(SPIFFS.largest_backend_write <= 502); // background selected at Configure assert(SPIFFS.largest_write == mesh::storage::CONTACT_RECORD_SIZE); assert(SPIFFS.removes == 0); // first publication has no backup/temp to delete assert(h.copies >= h.contacts.size()); @@ -196,14 +196,16 @@ void backgroundCRCVerificationUsesBoundedBatches() { if (read_count != 0) ++crc_read_passes; max_reads = std::max(max_reads, read_count); max_backend_reads = std::max(max_backend_reads, backend_count); - assert(read_count <= 8 && backend_count <= 4); + assert(read_count <= 8 && backend_count <= 1); } // These are actual production DataStore calls, not a direct transaction // benchmark. Default one-chunk verification fails this cadence witness. assert(SPIFFS.reads == 825); assert(crc_read_passes == 104); - assert(max_reads == 8 && max_backend_reads == 4); - assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 128); + assert(max_reads == 8 && max_backend_reads == 1); + assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 512); + assert(SPIFFS.backend_reads == 104); + assert(SPIFFS.backend_writes == 106); // actual347-contact background502 job assert(SPIFFS.files.at("/contacts3") == image(h)); bootCheck(h); } @@ -324,7 +326,7 @@ void deferredBeginFaultsAndSynchronousDefaults() { } void mutationDuringEveryBeginStage() { - for (unsigned completed = 0; completed <= 7; ++completed) { + for (unsigned completed = 0; completed <= 8; ++completed) { SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true; Host h = table(); @@ -345,8 +347,8 @@ void mutationDuringEveryBeginStage() { assert(store.serviceContactWrites(&h, nullptr)); // guard precedes every begin step assert(store.hasPendingContactWrites()); assert(SPIFFS.opens == opens && SPIFFS.buffer_config_calls == configurations); - if (completed < 6) assert(SPIFFS.startupOperations() == before_cancel); - if (completed < 5) assert(SPIFFS.files.at("/contacts3.tmp") == stale); + if (completed < 7) assert(SPIFFS.startupOperations() == before_cancel); + if (completed < 6) assert(SPIFFS.files.at("/contacts3.tmp") == stale); else assert(!SPIFFS.exists("/contacts3.tmp")); assert(SPIFFS.files.at("/contacts3") == old); drain(store, h); @@ -474,8 +476,9 @@ void realBackupCleanupFailure() { assert(!success && store.hasPendingContactWrites()); assert(SPIFFS.files.at("/contacts3") == old); assert(SPIFFS.files.at("/contacts3.bak") == std::vector({42})); - // Best-effort temp cleanup cannot hide the primary remove failure. - assert(SPIFFS.exists("/contacts3.tmp") && SPIFFS.missing_remove_logs == 0); + // Retirement fails before this transaction owns or opens a new temp. + assert(!SPIFFS.exists("/contacts3.tmp") && SPIFFS.missing_remove_logs == 0); + assert(SPIFFS.writes == 0 && SPIFFS.opens == 0 && !store.activeJob()); SPIFFS.fail_remove = false; drain(store, h); bootCheck(h); @@ -617,10 +620,46 @@ void boundedCancellationFlush() { } } + +void deferredCleanupThroughActualDataStore() { + SPIFFS = FakeFilesystem(); + SPIFFS.emulate_stdio = true; + Host h = table(32); + const auto old = image(h); + const std::vector older = {9, 8, 7}; + SPIFFS.files["/contacts3"] = old; + SPIFFS.files["/contacts3.bak"] = older; + DataStore store(SPIFFS); + h.contacts[0] = contact(904); + assert(store.markContactDirty(h.contacts[0])); + assert(store.serviceContactWrites(&h, nullptr)); // allocation only + for (unsigned step = 0; step < 3; ++step) + assert(store.serviceContactWrites(&h, nullptr)); // Target, Backup, Recover + assert(SPIFFS.files.at("/contacts3.bak") == older); + const unsigned before = SPIFFS.startupOperations(); + assert(store.serviceContactWrites(&h, nullptr)); // standalone RetireBackup + assert(SPIFFS.startupOperations() - before == 1); + assert(SPIFFS.files.at("/contacts3") == old && !SPIFFS.exists("/contacts3.bak")); + drain(store, h); + assert(SPIFFS.exists("/contacts3.bak")); + assert(!store.hasPendingContactWrites() && SPIFFS.files.at("/contacts3.bak") == old); + // A mutation after the completed job must retire that backup before replacing + // the newest authoritative target, with no stale index publication. + const auto prior = image(h); + h.contacts.erase(h.contacts.begin() + 4); + assert(store.releaseContact(contact(4))); + h.contacts[0] = contact(905); + assert(store.markContactDirty(h.contacts[0])); + drain(store, h); + assert(SPIFFS.files.at("/contacts3.bak") == prior); + bootCheck(h); +} + int main() { #if defined(CONTACT_BUFFER_BENCHMARK) bufferedAdmissionBenchmark(); #else + deferredCleanupThroughActualDataStore(); boundedFullTableAndFirstSave(); backgroundCRCVerificationUsesBoundedBatches(); boundedCancellationFlush(); diff --git a/test/fixtures/contact_publication/test.cpp b/test/fixtures/contact_publication/test.cpp new file mode 100644 index 000000000..7962fd72a --- /dev/null +++ b/test/fixtures/contact_publication/test.cpp @@ -0,0 +1,156 @@ +#include "FakeFilesystem.h" +#define FILESYSTEM FakeFilesystem +#define private public +#include "transaction_under_test.h" +#undef private +#include +using Writer=mesh::ContactFileTransaction; +using Progress=Writer::CommitProgress; +using Bytes=std::vector; +#if defined(ESP32_PLATFORM) +#define DEFER_BEGIN ,true +#else +#define DEFER_BEGIN +#endif +Bytes image(size_t n,uint8_t salt=0){Bytes out(n);for(size_t i=0;imetadata_probes;if(fs->metadata_error)return false;found=fs->exists(name);return true;} +void begin(Writer& w,bool coalesce=false){ +#if defined(ESP32_PLATFORM) + for(unsigned i=0;!w&&i<9;++i){const auto before=SPIFFS.startupOperations();assert(w.serviceBegin(coalesce)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-before<=1);} +#else + (void)coalesce; +#endif + assert(w); +} +void write(Writer& w,const Bytes& bytes){for(size_t i=0;i1?argv[1]:"all";if(mode=="contracts")contracts();else if(mode=="retire")retirement();else if(mode=="cancel")repeatedAndCancel();else if(mode=="faults")faultsAndCrash();else if(mode=="counts")counts();else assert(false);} diff --git a/test/fixtures/contact_writer_batch/test.cpp b/test/fixtures/contact_writer_batch/test.cpp new file mode 100644 index 000000000..4cbeb7d75 --- /dev/null +++ b/test/fixtures/contact_writer_batch/test.cpp @@ -0,0 +1,173 @@ +// Executes the real CFT with the existing stdio-admission model. +#include "FakeFilesystem.h" +#define FILESYSTEM FakeFilesystem +#include "transaction_under_test.h" +#include + +using Writer = mesh::ContactFileTransaction; +using Progress = Writer::CommitProgress; +using Bytes = std::vector; +Bytes image(size_t bytes) { + Bytes result(bytes); + for (size_t i = 0; i < bytes; ++i) result[i] = uint8_t(i * 23 + i / 7); + return result; +} +bool presence(FakeFilesystem* fs, const char* path, bool& found) { + ++fs->metadata_probes; + if (fs->metadata_error) return false; + found = fs->exists(path); + return true; +} +void begin(Writer& writer, bool background) { + for (unsigned i = 0; !writer && i < 8; ++i) + assert(writer.serviceBegin(background) != Writer::BeginProgress::Failed); + assert(writer); +} +void records(Writer& writer, const Bytes& bytes, size_t cap) { + assert(bytes.size() % 152 == 0); + for (size_t offset = 0; offset < bytes.size(); offset += 152) { + const unsigned before = SPIFFS.backend_writes; + assert(writer.write(bytes.data() + offset, 152) == 152); + assert(SPIFFS.backend_writes - before <= 1); + assert(SPIFFS.largest_backend_write <= cap); + } +} +Progress finish(Writer& writer, unsigned budget = 8) { + Progress progress = Progress::Pending; + for (unsigned i = 0; progress == Progress::Pending && i < 2000; ++i) + progress = writer.serviceCommit(true, budget); + assert(progress != Progress::Pending); + return progress; +} +void measure(unsigned count, bool background) { + SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true; + const Bytes bytes = image(count * 152); + Writer writer(&SPIFFS, "/contacts3", presence, true); + begin(writer, background); + const unsigned configurations = SPIFFS.buffer_config_calls; + assert(writer.serviceBegin(!background) == Writer::BeginProgress::Ready); + assert(SPIFFS.buffer_config_calls == configurations); // selection latched + const size_t cap = background ? 502 : 251; + records(writer, bytes, cap); + assert(SPIFFS.writes == count); + assert(SPIFFS.backend_writes == bytes.size() / cap); + assert(finish(writer) == Progress::Succeeded); + assert(SPIFFS.files.at("/contacts3") == bytes); + assert(SPIFFS.backend_writes == (bytes.size() + cap - 1) / cap); + assert(SPIFFS.largest_backend_write <= cap); + assert(SPIFFS.missing_remove_logs == 0); + printf("{\"records\":%u,\"background\":%s,\"bytes\":%zu,\"logical_writes\":%u," + "\"backend_writes\":%u,\"largest_backend_write\":%zu}\n", + count, background ? "true" : "false", bytes.size(), SPIFFS.writes, + SPIFFS.backend_writes, SPIFFS.largest_backend_write); +} +void synchronousDefault() { + SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true; + const Bytes bytes = image(347 * 152); + Writer writer(&SPIFFS, "/contacts3", presence); + assert(writer); records(writer, bytes, 251); + assert(writer.commit()); + assert(SPIFFS.backend_writes == 211 && SPIFFS.largest_backend_write == 251); + assert(SPIFFS.files.at("/contacts3") == bytes); +} +void configureTime(bool earlier, bool configure) { + SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true; + Writer writer(&SPIFFS, "/contacts3", presence, true); + // Reach the open stream without configuring it, regardless of whether + // background preparation also retires an earlier backup. + unsigned passes = 0; + while (SPIFFS.opens == 0) { + assert(++passes < 16); + assert(writer.serviceBegin(earlier) == Writer::BeginProgress::Pending); + } + assert(SPIFFS.buffer_config_calls == 0 && SPIFFS.writes == 0); + assert(writer.serviceBegin(configure) == Writer::BeginProgress::Ready); + const unsigned configurations = SPIFFS.buffer_config_calls; + assert(writer.serviceBegin(!configure) == Writer::BeginProgress::Ready); + assert(SPIFFS.buffer_config_calls == configurations); + records(writer, image(4 * 152), configure ? 502 : 251); + assert(SPIFFS.largest_backend_write == (configure ? 502 : 251)); +} +void cancellation() { + // gcd(152,502)=2:251 records are exactly76 full502 buffers. + for (bool background : {false,true}) for (unsigned n : {1U,2U,3U,4U,250U,251U,252U,347U}) { + SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true; + const Bytes old = image(152), next = image(n * 152); + SPIFFS.files["/contacts3"] = old; + const size_t cap = background ? 502 : 251; + unsigned before_close = 0; + { + Writer writer(&SPIFFS, "/contacts3", presence, true); begin(writer,background); + records(writer,next,cap); + assert(SPIFFS.backend_writes == next.size() / cap); + before_close = SPIFFS.backend_writes; + assert(SPIFFS.files.at("/contacts3") == old); + } + assert(SPIFFS.backend_writes-before_close == unsigned(next.size()%cap != 0)); + assert(SPIFFS.backend_writes == (next.size()+cap-1)/cap); + assert(SPIFFS.largest_backend_write <= cap); + assert(SPIFFS.files.at("/contacts3") == old && !SPIFFS.exists("/contacts3.tmp")); + } +} +void setupFailures() { + for (bool background : {false,true}) for (unsigned fault : {1U,2U,3U}) { + SPIFFS=FakeFilesystem(); SPIFFS.emulate_stdio=true; + const Bytes old=image(152); SPIFFS.files["/contacts3"]=old; + if (fault==1) SPIFFS.fail_buffer_config=1; + if (fault==2) SPIFFS.metadata_error=true; + if (fault==3) SPIFFS.fail_create=SPIFFS.partial_create_failure=true; + { + Writer writer(&SPIFFS,"/contacts3",presence,true); + auto result=Writer::BeginProgress::Pending; + for (unsigned i=0;result==Writer::BeginProgress::Pending && i<8;++i) + result=writer.serviceBegin(background); + assert(result==Writer::BeginProgress::Failed && !writer); + assert(writer.write(old.data(),old.size())==0); + assert(writer.serviceCommit()==Progress::Failed); + assert(SPIFFS.writes==0 && SPIFFS.backend_writes==0 && SPIFFS.reads==0); + } + assert(SPIFFS.files.at("/contacts3")==old); + if(fault!=2) assert(!SPIFFS.exists("/contacts3.tmp")); + } +} +void faultsAndCrash() { + for (bool background : {false,true}) for(unsigned fault : {0U,1U,2U,3U,4U,5U}) { + SPIFFS=FakeFilesystem(); SPIFFS.emulate_stdio=true; + const Bytes old=image(152),next=image(4*152);SPIFFS.files["/contacts3"]=old; + { + Writer writer(&SPIFFS,"/contacts3",presence,true);begin(writer,background); + if(fault==1) { + SPIFFS.max_write=151; + assert(writer.write(next.data(),152)==151); + } else records(writer,next,background?502:251); + if(fault==2) SPIFFS.fail_read="/contacts3.tmp"; + if(fault==3) SPIFFS.fail_rename=1; + if(fault==4) SPIFFS.fail_rename=2; + if(fault==5) { + assert(writer.serviceCommit()==Progress::Pending); // flush before corrupting temp + SPIFFS.files.at("/contacts3.tmp")[30]^=1; + } + const auto result=finish(writer); + assert(result==(fault==0?Progress::Succeeded:Progress::Failed)); + } + assert(SPIFFS.files.at("/contacts3")== (fault==0?next:old)); + assert(!SPIFFS.exists("/contacts3.tmp")); + const auto snapshots=SPIFFS.rename_snapshots; + for (const auto& snapshot:snapshots) { + SPIFFS=FakeFilesystem();SPIFFS.files=snapshot; + assert(Writer::recover(&SPIFFS,"/contacts3",presence)); + const auto& recovered=SPIFFS.files.at("/contacts3"); + assert(recovered==old || recovered==next); + } + } +} +int main(int argc,char** argv) { + const std::string mode=argc>1?argv[1]:"all"; + if(mode=="measure") { + for(unsigned n : {347U,349U,350U}) for(bool background : {false,true}) measure(n,background); + } else if(mode=="default") synchronousDefault(); + else if(mode=="configure") {configureTime(true,false);configureTime(false,true);} + else if(mode=="cancel") cancellation(); + else if(mode=="faults") {setupFailures();faultsAndCrash();} + else assert(false && "unknown fixture mode"); +} diff --git a/test/test_contact_cache.py b/test/test_contact_cache.py index bd3893a68..5550b6e05 100644 --- a/test/test_contact_cache.py +++ b/test/test_contact_cache.py @@ -91,10 +91,10 @@ class ContactCacheTest(unittest.TestCase): functions.append("bool DataStore::loadContactPages(") implementation += "\n".join(method(store, signature) for signature in functions) if disable_crc_batch: - self.assertIn("_contact_write->serviceCommit(true, 8)", implementation) + self.assertIn("_contact_write->serviceCommit(true, 8, true)", implementation) implementation = implementation.replace( - "_contact_write->serviceCommit(true, 8)", - "_contact_write->serviceCommit()") + "_contact_write->serviceCommit(true, 8, true)", + "_contact_write->serviceCommit(true, 1, true)") (temp / "store_under_test.h").write_text(implementation) packet = (ROOT / "src/Packet.cpp").read_text() (temp / "packet_under_test.h").write_text("namespace mesh {\n" + "\n".join( diff --git a/test/test_esp32_contact_crc_batch.py b/test/test_esp32_contact_crc_batch.py index 6e492c686..223340a54 100644 --- a/test/test_esp32_contact_crc_batch.py +++ b/test/test_esp32_contact_crc_batch.py @@ -9,7 +9,8 @@ import tempfile import unittest ROOT = Path(__file__).resolve().parents[1] -SIGNATURE = "CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) {" +SIGNATURE = ("CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1,\n" + " bool defer_backup_cleanup = false) {") OPAQUE_FILE = """#pragma once #include #include @@ -58,17 +59,29 @@ class ContactCRCBatchTest(unittest.TestCase): anchor = "ESP_VERIFY_BUFFER_SIZE = 128;" self.assertEqual(source.count(anchor), 1) source = source.replace(anchor, "ESP_VERIFY_BUFFER_SIZE = 4096;") + elif negative == "background_stdio": + anchor = "max_verify_chunks >= 8 ? 512 : ESP_VERIFY_BUFFER_SIZE" + self.assertEqual(source.count(anchor), 1) + source = source.replace( + anchor, "max_verify_chunks >= 8 ? 4096 : ESP_VERIFY_BUFFER_SIZE") + fixture = (ROOT / "test/fixtures/contact_crc_batch/test.cpp").read_text() + if negative == "background_stdio": + # Start with a large file so the negative executes a real >512B + # refill, rather than failing only the selected-size assertion. + self.assertEqual(fixture.count(" limitsAndTails();"), 1) + fixture = fixture.replace(" limitsAndTails();", " verify(347 * 152, 8);") # Count actual commit() calls without replacing its body or algorithm. source = source.replace(SIGNATURE, SIGNATURE + "\n ++fixture_commit_calls;") with tempfile.TemporaryDirectory(prefix="mesh-contact-crc-") as directory: temp = Path(directory) (temp / "transaction_under_test.h").write_text(source) + (temp / "fixture.cpp").write_text(fixture) binary = temp / "fixture" result = subprocess.run([ "c++", "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-fsanitize=address,undefined", "-fno-omit-frame-pointer", "-D" + platform + "=1", "-I", str(temp), "-I", str(ROOT / "src"), - str(ROOT / "test/fixtures/contact_crc_batch/test.cpp"), + str(temp / "fixture.cpp"), "-o", str(binary), ], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) @@ -76,7 +89,8 @@ class ContactCRCBatchTest(unittest.TestCase): if negative: self.assertNotEqual(result.returncode, 0, result.stdout) markers = {"clamp": "reads <= bound", "synchronous": "fixture_commit_calls == 829", - "stdio": "largest_backend_read <= 128"} + "stdio": "reader_buffer_size == selected", + "background_stdio": "backend_bytes <= 512"} self.assertIn(markers[negative], result.stderr) return self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -85,8 +99,10 @@ class ContactCRCBatchTest(unittest.TestCase): def test_esp_bounds_tails_failures_crc_cancellation_and_crash(self): result = self.run_native() self.assertEqual(result, {"one_passes": 829, "eight_passes": 108, - "logical_reads": 825, "backend_reads": 413, - "max_pass_reads": 8, "max_pass_refills": 4}) + "logical_reads": 825, "backend_reads": 104, + "max_pass_reads": 8, "max_pass_refills": 1, + "max_pass_backend_bytes": 512, "reader_buffer_size": 512, + "one_backend_reads": 413, "one_reader_buffer_size": 128}) def test_other_platform_ignores_requested_batch(self): result = self.run_native("RP2040_PLATFORM") @@ -103,6 +119,9 @@ class ContactCRCBatchTest(unittest.TestCase): def test_large_stdio_read_ahead_is_detected(self): self.run_native(negative="stdio") + def test_large_background_stdio_read_ahead_is_detected(self): + self.run_native(negative="background_stdio") + def test_opaque_xtensa_fixture_frame_and_object_sizes(self): compiler = os.environ.get("CONTACT_CRC_XTENSA_CXX") or shutil.which("xtensa-esp32s3-elf-g++") if not compiler: @@ -134,7 +153,7 @@ class ContactCRCBatchTest(unittest.TestCase): frames = [line.split("\t") for line in (temp / "frame.su").read_text().splitlines() if "ContactFileTransaction::serviceCommit(" in line] self.assertEqual(len(frames), 1) - self.assertEqual(int(frames[0][1]), 112) + self.assertEqual(int(frames[0][1]), 128) nm = str(Path(compiler).with_name("xtensa-esp32s3-elf-nm")) symbols = subprocess.check_output([nm, "-S", str(temp / "frame.o")], text=True) size = next(int(line.split()[1], 16) for line in symbols.splitlines() diff --git a/test/test_esp32_contact_publication.py b/test/test_esp32_contact_publication.py new file mode 100644 index 000000000..c5d1cfaa3 --- /dev/null +++ b/test/test_esp32_contact_publication.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +from pathlib import Path +import json,subprocess,tempfile,unittest +ROOT=Path(__file__).resolve().parents[1] +FIXTURE=ROOT/"test/fixtures/contact_publication/test.cpp" +class Publication(unittest.TestCase): + def run_native(self,mode,platform='ESP32_PLATFORM',negative=None): + source=(ROOT/'src/helpers/ContactFileTransaction.h').read_text().replace('#include "IdentityStore.h"','// native supplied File').replace('#include "PersistentStoreFormat.h"','#include ') + if negative=='cleanup': + a='if (!defer_backup_cleanup)';self.assertEqual(source.count(a),1);source=source.replace(a,'if (true)') + elif negative=='retirement': + a='ok = _fs->remove(_backup);';self.assertEqual(source.count(a),2);source=source.replace(a,'ok = true; // negative: leave previous backup',1) + with tempfile.TemporaryDirectory(prefix='mesh-publication-') as folder: + p=Path(folder);(p/'transaction_under_test.h').write_text(source);binary=p/'test' + result=subprocess.run(['c++','-std=c++17','-O1','-g','-Wall','-Wextra','-fsanitize=address,undefined','-fno-omit-frame-pointer','-D'+platform+'=1','-I',str(p),'-I',str(ROOT/'src'),'-I',str(ROOT/'test/fixtures/contact_cache'),'-I',str(ROOT/'test/fixtures/contact_cache/mocks'),str(FIXTURE),'-o',str(binary)],text=True,capture_output=True) + self.assertEqual(result.returncode,0,result.stderr) + result=subprocess.run([str(binary),mode],text=True,capture_output=True) + if negative: + self.assertNotEqual(result.returncode,0,result.stdout) + self.assertIn('SPIFFS.',result.stderr) + else:self.assertEqual(result.returncode,0,result.stdout+result.stderr) + def test_default_and_deferred_success_contracts(self):self.run_native('contracts') + def test_retirement_and_recovery_failure_before_temp_ownership(self):self.run_native('retire') + def test_repeated_transaction_all_begin_cancellation_boundaries(self):self.run_native('cancel') + def test_publish_failures_and_every_rename_crash(self):self.run_native('faults') + def test_251_502_counts_and_unchanged_commit_cadence(self):self.run_native('counts') + def test_other_platform_retains_cleanup_despite_true_arg(self):self.run_native('contracts','RP2040_PLATFORM') + def test_removed_defer_guard_is_detected(self):self.run_native('contracts',negative='cleanup') + def test_omitted_retirement_is_detected(self):self.run_native('retire',negative='retirement') +if __name__=='__main__':unittest.main() diff --git a/test/test_esp32_contact_writer_batch.py b/test/test_esp32_contact_writer_batch.py new file mode 100644 index 000000000..db27aeb6b --- /dev/null +++ b/test/test_esp32_contact_writer_batch.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Execute background502/default251 admission bounds on the real CFT.""" +from pathlib import Path +import json,os,subprocess,tempfile,unittest +ROOT=Path(os.environ.get("CONTACT_WRITER_ROOT",Path(__file__).resolve().parents[1])) +HEADER=Path(os.environ.get("CONTACT_WRITER_HEADER",ROOT/"src/helpers/ContactFileTransaction.h")) +DATASTORE=Path(os.environ.get("CONTACT_WRITER_DATASTORE",ROOT/"examples/companion_radio/DataStore.cpp")) +FIXTURE=Path(__file__).resolve().parent/"fixtures/contact_writer_batch/test.cpp" +class WriterBatch(unittest.TestCase): + def run_native(self,mode,negative=None): + source=HEADER.read_text().replace('#include "IdentityStore.h"','// native supplied FILESYSTEM').replace( + '#include "PersistentStoreFormat.h"','#include ') + if negative=="default": + anchor='while (serviceBegin() == BeginProgress::Pending) {}' + self.assertEqual(source.count(anchor),1) + source=source.replace(anchor,'while (serviceBegin(true) == BeginProgress::Pending) {}') + if negative=="large": + anchor='ESP_BACKGROUND_WRITE_BUFFER_SIZE = 502;' + self.assertEqual(source.count(anchor),1);source=source.replace(anchor,'ESP_BACKGROUND_WRITE_BUFFER_SIZE = 4096;') + # Reuse the existing immediate-full-newlib stdio model unchanged. + with tempfile.TemporaryDirectory(prefix='mesh-writer-batch-') as directory: + tmp=Path(directory);(tmp/'transaction_under_test.h').write_text(source) + binary=tmp/'fixture' + cmd=['c++','-std=c++17','-O1','-g','-Wall','-Wextra','-fsanitize=address,undefined', + '-fno-omit-frame-pointer','-DESP32_PLATFORM=1','-I',str(tmp),'-I',str(ROOT/'src'), + '-I',str(ROOT/'test/fixtures/contact_cache'),'-I',str(ROOT/'test/fixtures/contact_cache/mocks'), + str(FIXTURE),'-o',str(binary)] + compiled=subprocess.run(cmd,text=True,capture_output=True) + self.assertEqual(compiled.returncode,0,compiled.stderr) + result=subprocess.run([str(binary),mode],text=True,capture_output=True) + if negative: + self.assertNotEqual(result.returncode,0,result.stdout) + self.assertIn('largest_backend_write <= cap',result.stderr) + return + self.assertEqual(result.returncode,0,result.stdout+result.stderr) + return [json.loads(line) for line in result.stdout.splitlines()] + def test_counts_admissions_and_exact_committed_bytes(self): + rows=self.run_native('measure');self.assertEqual(len(rows),6) + for row in rows: + n=row['records'];cap=502 if row['background'] else 251 + self.assertEqual(row['logical_writes'],n) + self.assertEqual(row['bytes'],n*152) + self.assertEqual(row['backend_writes'],(n*152+cap-1)//cap) + self.assertEqual(row['largest_backend_write'],cap) + self.assertEqual([(r['backend_writes']) for r in rows],[211,106,212,106,212,106]) + def test_synchronous_constructor_keeps_default251(self):self.run_native('default') + def test_configure_time_selection_is_latched(self):self.run_native('configure') + def test_cancellation_tail_and_old_target(self):self.run_native('cancel') + def test_fail_closed_config_short_write_corruption_and_crash(self):self.run_native('faults') + def test_old_default_contract_regression_is_detected(self):self.run_native('default','default') + def test_4096_burst_is_detected(self):self.run_native('measure','large') +if __name__=='__main__':unittest.main()