mirror of
https://github.com/mikecarper/MeshCore.git
synced 2026-10-20 13:52:10 +00:00
Coalesce ESP background contact I/O and defer backup cleanup
This commit is contained in:
mikecarper
@@ -2531,7 +2531,7 @@ bool DataStore::serviceContactWrite(DataStoreHost* host,
|
||||
// The revision/session guard above also applies to every setup step.
|
||||
// Keep initialization separate from serialization, including its final
|
||||
// buffer configuration, so transports are serviced between operations.
|
||||
const auto progress = _contact_write->serviceBegin();
|
||||
const auto progress = _contact_write->serviceBegin(true);
|
||||
if (progress == mesh::ContactFileTransaction::BeginProgress::Failed) {
|
||||
cancelContactWrite();
|
||||
return false;
|
||||
@@ -2581,7 +2581,7 @@ bool DataStore::serviceContactWrite(DataStoreHost* host,
|
||||
#endif
|
||||
// Keep the 64-byte CRC scratch while verifying at most 512 bytes per
|
||||
// background pass; transports run between these bounded batches.
|
||||
const auto progress = _contact_write->serviceCommit(true, 8);
|
||||
const auto progress = _contact_write->serviceCommit(true, 8, true);
|
||||
if (progress == mesh::ContactFileTransaction::CommitProgress::Pending) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -33,11 +33,12 @@ private:
|
||||
// This bounded stdio buffer coalesces records without the SDK's 4 KiB burst;
|
||||
// one logical record can flush at most one full buffer per write pass.
|
||||
static constexpr size_t ESP_WRITE_BUFFER_SIZE = 251;
|
||||
static constexpr size_t ESP_BACKGROUND_WRITE_BUFFER_SIZE = 502;
|
||||
static constexpr size_t ESP_VERIFY_BUFFER_SIZE = 128;
|
||||
static_assert(storage::CONTACT_RECORD_SIZE <= ESP_WRITE_BUFFER_SIZE,
|
||||
"Contact records must fit one bounded writer buffer");
|
||||
enum class BeginStage : uint8_t {
|
||||
Target, Backup, Recover, Temp, Remove, Open, Configure, Ready, Failed
|
||||
Target, Backup, Recover, RetireBackup, Temp, Remove, Open, Configure, Ready, Failed
|
||||
};
|
||||
BeginStage _begin_stage = BeginStage::Target;
|
||||
bool _begin_target_exists = false;
|
||||
@@ -106,7 +107,7 @@ public:
|
||||
operator bool() const { return _ok; }
|
||||
#if defined(ESP32_PLATFORM)
|
||||
enum class BeginProgress : uint8_t { Pending, Ready, Failed };
|
||||
BeginProgress serviceBegin() {
|
||||
BeginProgress serviceBegin(bool background_contact_write = false) {
|
||||
bool ok = true;
|
||||
switch (_begin_stage) {
|
||||
case BeginStage::Target:
|
||||
@@ -118,8 +119,22 @@ public:
|
||||
if (ok) _begin_stage = BeginStage::Recover;
|
||||
break;
|
||||
case BeginStage::Recover:
|
||||
if (!_begin_target_exists && _begin_backup_exists)
|
||||
if (!_begin_target_exists && _begin_backup_exists) {
|
||||
ok = _fs->rename(_backup, _target);
|
||||
if (ok) {
|
||||
_begin_target_exists = true;
|
||||
_begin_backup_exists = false;
|
||||
}
|
||||
}
|
||||
if (ok) _begin_stage = background_contact_write
|
||||
? BeginStage::RetireBackup : BeginStage::Temp;
|
||||
break;
|
||||
case BeginStage::RetireBackup:
|
||||
// A successful prior publication may retain its old backup. Retire it
|
||||
// in a separate setup pass only when the original target was present.
|
||||
// Recover already consumed the backup when the target was missing.
|
||||
if (_begin_target_exists && _begin_backup_exists)
|
||||
ok = _fs->remove(_backup);
|
||||
if (ok) _begin_stage = BeginStage::Temp;
|
||||
break;
|
||||
case BeginStage::Temp:
|
||||
@@ -140,9 +155,11 @@ public:
|
||||
if (ok) _begin_stage = BeginStage::Configure;
|
||||
break;
|
||||
case BeginStage::Configure:
|
||||
// Arduino otherwise batches small writes into a 4 KiB stdio buffer.
|
||||
// Configure the newly opened stream before its first I/O.
|
||||
ok = _file.setBufferSize(ESP_WRITE_BUFFER_SIZE);
|
||||
// Select once at Configure, before the stream's first I/O. Default
|
||||
// and synchronous callers retain one 251-byte SPIFFS payload page;
|
||||
// background contacts coalesce two pages without a larger record.
|
||||
ok = _file.setBufferSize(background_contact_write
|
||||
? ESP_BACKGROUND_WRITE_BUFFER_SIZE : ESP_WRITE_BUFFER_SIZE);
|
||||
if (ok) {
|
||||
_ok = true;
|
||||
_begin_stage = BeginStage::Ready;
|
||||
@@ -176,7 +193,8 @@ public:
|
||||
// callers may request up to eight chunks while reusing the same scratch
|
||||
// buffer. Transports run between passes without exposing the rename gap;
|
||||
// synchronous commit() and other platforms retain the one-chunk default.
|
||||
CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) {
|
||||
CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1,
|
||||
bool defer_backup_cleanup = false) {
|
||||
#if defined(ESP32_PLATFORM)
|
||||
// A premature commit must not publish, clean up, or implicitly aggregate
|
||||
// the deferred setup operations inside this call.
|
||||
@@ -193,11 +211,13 @@ public:
|
||||
_verify = _fs->open(_temp, "r");
|
||||
ok = static_cast<bool>(_verify);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
// fread() otherwise reads ahead up to 4 KiB while the caller requests
|
||||
// only one 64-byte CRC chunk. A bounded two-chunk read-ahead buffer
|
||||
// halves backend reads without growing that scratch buffer. The caller's
|
||||
// separate chunk budget bounds each pass. Configuration must succeed.
|
||||
ok = ok && _verify.setBufferSize(ESP_VERIFY_BUFFER_SIZE);
|
||||
// Select read-ahead at Open only. Default/synchronous callers retain
|
||||
// 128 bytes; an eight-chunk background pass may coalesce its 512-byte
|
||||
// budget in one backend refill. Later budget changes retain this choice,
|
||||
// with at most 512 backend bytes per pass and the same 64-byte scratch.
|
||||
// Configuration must succeed before verification or publication.
|
||||
ok = ok && _verify.setBufferSize(
|
||||
max_verify_chunks >= 8 ? 512 : ESP_VERIFY_BUFFER_SIZE);
|
||||
#endif
|
||||
ok = ok && _verify.size() == _size;
|
||||
if (ok) {
|
||||
@@ -252,8 +272,16 @@ public:
|
||||
if (!ok && backed_up) _fs->rename(_backup, _target);
|
||||
// Cleanup remains best effort after the publish/rollback result, but an
|
||||
// absent file must not trigger the SDK's error log on the shared USB port.
|
||||
if (ok) removeIfPresent(_fs, _backup, _presence);
|
||||
else removeIfPresent(_fs, _temp, _presence);
|
||||
if (ok) {
|
||||
#if defined(ESP32_PLATFORM)
|
||||
// Background contacts publish the target and its path handles now.
|
||||
// A later transaction retires this recoverable old backup separately.
|
||||
if (!defer_backup_cleanup)
|
||||
#else
|
||||
(void)defer_backup_cleanup; // Other platforms retain immediate cleanup.
|
||||
#endif
|
||||
removeIfPresent(_fs, _backup, _presence);
|
||||
} else removeIfPresent(_fs, _temp, _presence);
|
||||
_finished = true;
|
||||
return ok ? CommitProgress::Succeeded : CommitProgress::Failed;
|
||||
}
|
||||
|
||||
+23
-2
@@ -489,11 +489,11 @@ static void background_crc_verification_uses_bounded_batches() {
|
||||
total_crc_reads += read_count;
|
||||
max_reads = std::max(max_reads, read_count);
|
||||
max_backend_reads = std::max(max_backend_reads, backend_count);
|
||||
assert(read_count <= 8 && backend_count <= 4);
|
||||
assert(read_count <= 8 && backend_count <= 1);
|
||||
}
|
||||
assert(total_crc_reads == 825);
|
||||
assert(crc_read_passes == 104);
|
||||
assert(max_reads == 8 && max_backend_reads == 4);
|
||||
assert(max_reads == 8 && max_backend_reads == 1);
|
||||
assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
f.host.contacts.clear();
|
||||
f.store.loadContacts(&f.host);
|
||||
@@ -818,9 +818,30 @@ static void contact_tx_policy_round_trips_without_growing_records() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#if defined(ESP32_PLATFORM)
|
||||
static void deferred_cleanup_publishes_actual_cached_handles() {
|
||||
Fixture f(40);
|
||||
const auto old = SPIFFS.files.at("/contacts3");
|
||||
SPIFFS.files["/contacts3.bak"] = {9, 8, 7};
|
||||
assert(f.host.contacts[0].setRawPath(route(2100).data()));
|
||||
assert(f.store.markContactDirty(f.host.contacts[0]));
|
||||
for (unsigned step = 0; step < 4; ++step)
|
||||
assert(f.store.serviceContactWrites(&f.host, cachedContactFilter));
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == std::vector<uint8_t>({9, 8, 7}));
|
||||
assert(f.store.serviceContactWrites(&f.host, cachedContactFilter));
|
||||
assert(!SPIFFS.exists("/contacts3.bak") && SPIFFS.files.at("/contacts3") == old);
|
||||
drain_cooperative_save(f);
|
||||
assert(!f.store.hasPendingContactWrites() && SPIFFS.files.at("/contacts3.bak") == old);
|
||||
f.check(0, route(2100));
|
||||
for (unsigned i = 1; i < 40; ++i) f.check(i, route(i));
|
||||
}
|
||||
#endif
|
||||
|
||||
int main() {
|
||||
contact_tx_policy_round_trips_without_growing_records();
|
||||
#if defined(ESP32_PLATFORM)
|
||||
deferred_cleanup_publishes_actual_cached_handles();
|
||||
cooperative_save_services_commands_and_preserves_snapshots();
|
||||
background_crc_verification_uses_bounded_batches();
|
||||
cooperative_mutations_never_publish_obsolete_indices();
|
||||
|
||||
+28
-6
@@ -41,6 +41,7 @@ public:
|
||||
unsigned config_calls = 0, closes = 0, renames = 0;
|
||||
unsigned fail_config = 0, short_read_call = 0, fail_rename = 0;
|
||||
size_t largest_read = 0, largest_backend_read = 0, max_write = SIZE_MAX;
|
||||
size_t backend_read_bytes = 0, reader_buffer_size = 0;
|
||||
bool fail_open_read = false, fail_open_write = false;
|
||||
std::vector<Files> rename_snapshots;
|
||||
bool exists(const char* path) { return files.count(path) != 0; }
|
||||
@@ -65,6 +66,7 @@ bool File::setBufferSize(size_t size) {
|
||||
if (++stream->fs->config_calls == stream->fs->fail_config) return false;
|
||||
assert(stream->buffer.empty() && stream->position == 0);
|
||||
stream->buffer_size = size;
|
||||
if (!stream->writing) stream->fs->reader_buffer_size = size;
|
||||
return true;
|
||||
}
|
||||
size_t File::size() const { return stream ? stream->fs->files.at(stream->path).size() : 0; }
|
||||
@@ -104,6 +106,7 @@ size_t File::read(uint8_t* data, size_t length) {
|
||||
const size_t n = std::min(stream->buffer_size, available);
|
||||
if (!n) break;
|
||||
++fs.backend_reads;
|
||||
fs.backend_read_bytes += n;
|
||||
fs.largest_backend_read = std::max(fs.largest_backend_read, n);
|
||||
stream->buffer.assign(source.begin() + stream->backend_position,
|
||||
source.begin() + stream->backend_position + n);
|
||||
@@ -147,6 +150,7 @@ unsigned effectiveBudget(unsigned requested) {
|
||||
|
||||
struct Measurement {
|
||||
unsigned passes, logical_reads, backend_reads, max_pass_reads, max_pass_refills;
|
||||
size_t max_pass_backend_bytes, reader_buffer_size;
|
||||
};
|
||||
Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
|
||||
FileSystem fs;
|
||||
@@ -156,19 +160,28 @@ Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
|
||||
assert(writer);
|
||||
writeImage(writer, next);
|
||||
unsigned passes = 0, max_reads = 0, max_refills = 0, data_passes = 0;
|
||||
size_t max_backend_bytes = 0;
|
||||
#if defined(ESP32_PLATFORM)
|
||||
// Selection is made once in Open, independently of the later read loop.
|
||||
const size_t selected = !use_default && budget >= 8 ? 512 : 128;
|
||||
#endif
|
||||
Progress result = Progress::Pending;
|
||||
while (result == Progress::Pending) {
|
||||
const unsigned before_reads = fs.reads, before_backend = fs.backend_reads;
|
||||
const size_t before_backend_bytes = fs.backend_read_bytes;
|
||||
const bool was_ready = writer.readyToPublish();
|
||||
result = use_default ? writer.serviceCommit() : writer.serviceCommit(true, budget);
|
||||
assert(++passes < 10000);
|
||||
const unsigned reads = fs.reads - before_reads, refills = fs.backend_reads - before_backend;
|
||||
max_reads = std::max(max_reads, reads);
|
||||
max_refills = std::max(max_refills, refills);
|
||||
const size_t backend_bytes = fs.backend_read_bytes - before_backend_bytes;
|
||||
max_backend_bytes = std::max(max_backend_bytes, backend_bytes);
|
||||
const unsigned bound = use_default ? 1 : effectiveBudget(budget);
|
||||
assert(reads <= bound);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(refills <= (bound + 1) / 2);
|
||||
assert(refills <= (64 * bound + selected - 1) / selected);
|
||||
assert(backend_bytes <= 512); // FILE refill admissions, not NOR operations or time
|
||||
#endif
|
||||
if (reads) {
|
||||
++data_passes;
|
||||
@@ -185,10 +198,12 @@ Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
|
||||
assert(fs.reads == (bytes + 63) / 64);
|
||||
assert(fs.largest_read <= 64);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(fs.largest_backend_read <= 128);
|
||||
assert(fs.backend_reads == (bytes + 127) / 128);
|
||||
assert(fs.reader_buffer_size == selected);
|
||||
assert(fs.largest_backend_read <= selected);
|
||||
assert(fs.backend_reads == (bytes + selected - 1) / selected);
|
||||
#endif
|
||||
return {passes, fs.reads, fs.backend_reads, max_reads, max_refills};
|
||||
return {passes, fs.reads, fs.backend_reads, max_reads, max_refills,
|
||||
max_backend_bytes, fs.reader_buffer_size};
|
||||
}
|
||||
void limitsAndTails() {
|
||||
for (size_t bytes : {size_t(0), size_t(1), size_t(63), size_t(64), size_t(65),
|
||||
@@ -203,6 +218,9 @@ void limitsAndTails() {
|
||||
fixture_commit_calls = 0;
|
||||
assert(writer.commit());
|
||||
assert(fixture_commit_calls == 829); // actual commit() drains default-one passes
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(fs.reader_buffer_size == 128 && fs.backend_reads == 413);
|
||||
#endif
|
||||
}
|
||||
void readFailureAndCorruption() {
|
||||
const auto old = image(19), next = image(1536);
|
||||
@@ -300,7 +318,11 @@ int main() {
|
||||
#endif
|
||||
const auto one = verify(347 * 152, 1), eight = verify(347 * 152, 8);
|
||||
printf("{\"one_passes\":%u,\"eight_passes\":%u,\"logical_reads\":%u,"
|
||||
"\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u}\n",
|
||||
"\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u,"
|
||||
"\"max_pass_backend_bytes\":%zu,\"reader_buffer_size\":%zu,"
|
||||
"\"one_backend_reads\":%u,\"one_reader_buffer_size\":%zu}\n",
|
||||
one.passes, eight.passes, eight.logical_reads, eight.backend_reads,
|
||||
eight.max_pass_reads, eight.max_pass_refills);
|
||||
eight.max_pass_reads, eight.max_pass_refills,
|
||||
eight.max_pass_backend_bytes, eight.reader_buffer_size,
|
||||
one.backend_reads, one.reader_buffer_size);
|
||||
}
|
||||
|
||||
+50
-11
@@ -137,7 +137,7 @@ unsigned drain(DataStore& store, Host& h, Filter filter = nullptr) {
|
||||
assert(SPIFFS.writes - writes <= 1);
|
||||
assert(SPIFFS.reads - reads <= 8);
|
||||
assert(SPIFFS.backend_writes - backend_writes <= 1);
|
||||
assert(SPIFFS.backend_reads - backend_reads <= 4);
|
||||
assert(SPIFFS.backend_reads - backend_reads <= 1);
|
||||
// Transport service runs here, between every production pass. It must
|
||||
// never observe an unpublished replacement or a target-name gap.
|
||||
if (store.hasPendingContactWrites()) {
|
||||
@@ -171,8 +171,8 @@ void boundedFullTableAndFirstSave() {
|
||||
const auto passes = drain(store, h, keepContact);
|
||||
assert(passes > h.contacts.size()); // serialization and CRC both yield
|
||||
assert(SPIFFS.largest_read <= 64);
|
||||
assert(SPIFFS.largest_backend_read <= 128);
|
||||
assert(SPIFFS.largest_backend_write <= 251);
|
||||
assert(SPIFFS.largest_backend_read <= 512); // background selected at Open
|
||||
assert(SPIFFS.largest_backend_write <= 502); // background selected at Configure
|
||||
assert(SPIFFS.largest_write == mesh::storage::CONTACT_RECORD_SIZE);
|
||||
assert(SPIFFS.removes == 0); // first publication has no backup/temp to delete
|
||||
assert(h.copies >= h.contacts.size());
|
||||
@@ -196,14 +196,16 @@ void backgroundCRCVerificationUsesBoundedBatches() {
|
||||
if (read_count != 0) ++crc_read_passes;
|
||||
max_reads = std::max(max_reads, read_count);
|
||||
max_backend_reads = std::max(max_backend_reads, backend_count);
|
||||
assert(read_count <= 8 && backend_count <= 4);
|
||||
assert(read_count <= 8 && backend_count <= 1);
|
||||
}
|
||||
// These are actual production DataStore calls, not a direct transaction
|
||||
// benchmark. Default one-chunk verification fails this cadence witness.
|
||||
assert(SPIFFS.reads == 825);
|
||||
assert(crc_read_passes == 104);
|
||||
assert(max_reads == 8 && max_backend_reads == 4);
|
||||
assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 128);
|
||||
assert(max_reads == 8 && max_backend_reads == 1);
|
||||
assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 512);
|
||||
assert(SPIFFS.backend_reads == 104);
|
||||
assert(SPIFFS.backend_writes == 106); // actual347-contact background502 job
|
||||
assert(SPIFFS.files.at("/contacts3") == image(h));
|
||||
bootCheck(h);
|
||||
}
|
||||
@@ -324,7 +326,7 @@ void deferredBeginFaultsAndSynchronousDefaults() {
|
||||
}
|
||||
|
||||
void mutationDuringEveryBeginStage() {
|
||||
for (unsigned completed = 0; completed <= 7; ++completed) {
|
||||
for (unsigned completed = 0; completed <= 8; ++completed) {
|
||||
SPIFFS = FakeFilesystem();
|
||||
SPIFFS.emulate_stdio = true;
|
||||
Host h = table();
|
||||
@@ -345,8 +347,8 @@ void mutationDuringEveryBeginStage() {
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // guard precedes every begin step
|
||||
assert(store.hasPendingContactWrites());
|
||||
assert(SPIFFS.opens == opens && SPIFFS.buffer_config_calls == configurations);
|
||||
if (completed < 6) assert(SPIFFS.startupOperations() == before_cancel);
|
||||
if (completed < 5) assert(SPIFFS.files.at("/contacts3.tmp") == stale);
|
||||
if (completed < 7) assert(SPIFFS.startupOperations() == before_cancel);
|
||||
if (completed < 6) assert(SPIFFS.files.at("/contacts3.tmp") == stale);
|
||||
else assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
assert(SPIFFS.files.at("/contacts3") == old);
|
||||
drain(store, h);
|
||||
@@ -474,8 +476,9 @@ void realBackupCleanupFailure() {
|
||||
assert(!success && store.hasPendingContactWrites());
|
||||
assert(SPIFFS.files.at("/contacts3") == old);
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == std::vector<uint8_t>({42}));
|
||||
// Best-effort temp cleanup cannot hide the primary remove failure.
|
||||
assert(SPIFFS.exists("/contacts3.tmp") && SPIFFS.missing_remove_logs == 0);
|
||||
// Retirement fails before this transaction owns or opens a new temp.
|
||||
assert(!SPIFFS.exists("/contacts3.tmp") && SPIFFS.missing_remove_logs == 0);
|
||||
assert(SPIFFS.writes == 0 && SPIFFS.opens == 0 && !store.activeJob());
|
||||
SPIFFS.fail_remove = false;
|
||||
drain(store, h);
|
||||
bootCheck(h);
|
||||
@@ -617,10 +620,46 @@ void boundedCancellationFlush() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void deferredCleanupThroughActualDataStore() {
|
||||
SPIFFS = FakeFilesystem();
|
||||
SPIFFS.emulate_stdio = true;
|
||||
Host h = table(32);
|
||||
const auto old = image(h);
|
||||
const std::vector<uint8_t> older = {9, 8, 7};
|
||||
SPIFFS.files["/contacts3"] = old;
|
||||
SPIFFS.files["/contacts3.bak"] = older;
|
||||
DataStore store(SPIFFS);
|
||||
h.contacts[0] = contact(904);
|
||||
assert(store.markContactDirty(h.contacts[0]));
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // allocation only
|
||||
for (unsigned step = 0; step < 3; ++step)
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // Target, Backup, Recover
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == older);
|
||||
const unsigned before = SPIFFS.startupOperations();
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // standalone RetireBackup
|
||||
assert(SPIFFS.startupOperations() - before == 1);
|
||||
assert(SPIFFS.files.at("/contacts3") == old && !SPIFFS.exists("/contacts3.bak"));
|
||||
drain(store, h);
|
||||
assert(SPIFFS.exists("/contacts3.bak"));
|
||||
assert(!store.hasPendingContactWrites() && SPIFFS.files.at("/contacts3.bak") == old);
|
||||
// A mutation after the completed job must retire that backup before replacing
|
||||
// the newest authoritative target, with no stale index publication.
|
||||
const auto prior = image(h);
|
||||
h.contacts.erase(h.contacts.begin() + 4);
|
||||
assert(store.releaseContact(contact(4)));
|
||||
h.contacts[0] = contact(905);
|
||||
assert(store.markContactDirty(h.contacts[0]));
|
||||
drain(store, h);
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == prior);
|
||||
bootCheck(h);
|
||||
}
|
||||
|
||||
int main() {
|
||||
#if defined(CONTACT_BUFFER_BENCHMARK)
|
||||
bufferedAdmissionBenchmark();
|
||||
#else
|
||||
deferredCleanupThroughActualDataStore();
|
||||
boundedFullTableAndFirstSave();
|
||||
backgroundCRCVerificationUsesBoundedBatches();
|
||||
boundedCancellationFlush();
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
#include "FakeFilesystem.h"
|
||||
#define FILESYSTEM FakeFilesystem
|
||||
#define private public
|
||||
#include "transaction_under_test.h"
|
||||
#undef private
|
||||
#include <cstdio>
|
||||
using Writer=mesh::ContactFileTransaction;
|
||||
using Progress=Writer::CommitProgress;
|
||||
using Bytes=std::vector<uint8_t>;
|
||||
#if defined(ESP32_PLATFORM)
|
||||
#define DEFER_BEGIN ,true
|
||||
#else
|
||||
#define DEFER_BEGIN
|
||||
#endif
|
||||
Bytes image(size_t n,uint8_t salt=0){Bytes out(n);for(size_t i=0;i<n;++i)out[i]=uint8_t(i*29+i/3+salt);return out;}
|
||||
bool presence(FakeFilesystem* fs,const char* name,bool& found){++fs->metadata_probes;if(fs->metadata_error)return false;found=fs->exists(name);return true;}
|
||||
void begin(Writer& w,bool coalesce=false){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
for(unsigned i=0;!w&&i<9;++i){const auto before=SPIFFS.startupOperations();assert(w.serviceBegin(coalesce)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-before<=1);}
|
||||
#else
|
||||
(void)coalesce;
|
||||
#endif
|
||||
assert(w);
|
||||
}
|
||||
void write(Writer& w,const Bytes& bytes){for(size_t i=0;i<bytes.size();i+=152){size_t n=std::min(size_t(152),bytes.size()-i);assert(w.write(bytes.data()+i,n)==n);}}
|
||||
unsigned finish(Writer& w,bool defer,unsigned budget=8){Progress result=Progress::Pending;unsigned calls=0;while(result==Progress::Pending){assert(++calls<2000);result=w.serviceCommit(true,budget,defer);}assert(result==Progress::Succeeded);return calls;}
|
||||
void reset(const Bytes& old){SPIFFS=FakeFilesystem();SPIFFS.emulate_stdio=true;SPIFFS.files["/contacts3"]=old;}
|
||||
void contracts(){
|
||||
const auto old=image(304,1),next=image(608,2);
|
||||
for(bool deferred:{false,true}) for(bool coalesce:{false,true}) {
|
||||
reset(old);{
|
||||
Writer w(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(w,coalesce);write(w,next);
|
||||
const unsigned removes=SPIFFS.removes;
|
||||
finish(w,deferred);
|
||||
assert(SPIFFS.files.at("/contacts3")==next);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(SPIFFS.exists("/contacts3.bak")==deferred);
|
||||
assert(SPIFFS.removes-removes==unsigned(!deferred));
|
||||
#else
|
||||
assert(!SPIFFS.exists("/contacts3.bak"));assert(SPIFFS.removes-removes==1);
|
||||
#endif
|
||||
}
|
||||
#if defined(ESP32_PLATFORM)
|
||||
if(deferred)assert(SPIFFS.files.at("/contacts3.bak")==old);
|
||||
#endif
|
||||
assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
}
|
||||
// Default synchronous failures retain the pre-existing prior-generation backup.
|
||||
#if defined(ESP32_PLATFORM)
|
||||
reset(old);const auto older=image(152,9);SPIFFS.files["/contacts3.bak"]=older;SPIFFS.fail_buffer_config=1;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence);assert(!w);}
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&SPIFFS.files.at("/contacts3.bak")==older);
|
||||
// RecoverFALSE skips retirement even when cleanupTRUE at Publish.
|
||||
reset(old);SPIFFS.files["/contacts3.bak"]=older;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,false);assert(SPIFFS.files.at("/contacts3.bak")==older);write(w,next);finish(w,true);}
|
||||
assert(SPIFFS.files.at("/contacts3.bak")==old);
|
||||
// RecoverTRUE latches the retirement stage; ConfigureFALSE retains writer251.
|
||||
reset(old);SPIFFS.files["/contacts3.bak"]=older;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(w.serviceBegin(false)==Writer::BeginProgress::Pending);assert(!SPIFFS.exists("/contacts3.bak"));
|
||||
begin(w,false);write(w,next);finish(w,true);assert(SPIFFS.largest_backend_write==251);}
|
||||
#endif
|
||||
// Default synchronous commit still cleans the old backup.
|
||||
reset(old);{Writer w(&SPIFFS,"/contacts3",presence);assert(w);write(w,next);assert(w.commit());}
|
||||
assert(!SPIFFS.exists("/contacts3.bak")&&SPIFFS.files.at("/contacts3")==next);
|
||||
}
|
||||
void retirement(){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
const auto old=image(304,1),older=image(152,2);reset(old);SPIFFS.files["/contacts3.bak"]=older;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&SPIFFS.files.at("/contacts3.bak")==older);
|
||||
const auto removes=SPIFFS.removes,probes=SPIFFS.metadata_probes,opens=SPIFFS.opens;
|
||||
assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(SPIFFS.removes-removes==1&&SPIFFS.metadata_probes==probes&&SPIFFS.opens==opens);
|
||||
assert(!SPIFFS.exists("/contacts3.bak")&&SPIFFS.files.at("/contacts3")==old);
|
||||
}
|
||||
// Missing target recovers backup, updates flags and performs no removal.
|
||||
SPIFFS=FakeFilesystem();SPIFFS.files["/contacts3.bak"]=old;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(w._begin_target_exists&&!w._begin_backup_exists);
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&!SPIFFS.exists("/contacts3.bak"));
|
||||
const auto removes=SPIFFS.removes;
|
||||
assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(SPIFFS.removes==removes&&SPIFFS.missing_remove_logs==0);
|
||||
}
|
||||
// Failed retirement stops before new/temp ownership; prior temp is untouched.
|
||||
reset(old);SPIFFS.files["/contacts3.bak"]=older;SPIFFS.files["/contacts3.tmp"]={9,8,7};SPIFFS.fail_remove=true;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(w.serviceBegin(true)==Writer::BeginProgress::Failed&&!w&&!w._owns_temp);
|
||||
assert(!w.commit()&&w.write(old.data(),old.size())==0);
|
||||
assert(SPIFFS.opens==0&&SPIFFS.writes==0);
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&SPIFFS.files.at("/contacts3.bak")==older);
|
||||
assert(SPIFFS.files.at("/contacts3.tmp")==Bytes({9,8,7}));
|
||||
#endif
|
||||
}
|
||||
void repeatedAndCancel(){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
const auto old=image(152,1),next=image(608,2),last=image(760,3);reset(old);
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,true);write(w,next);finish(w,true);}
|
||||
assert(SPIFFS.files.at("/contacts3.bak")==old);
|
||||
for(unsigned completed=0;completed<=8;++completed) {
|
||||
reset(next);SPIFFS.files["/contacts3.bak"]=old;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<completed;++i){const auto n=SPIFFS.startupOperations();assert(w.serviceBegin(true)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-n<=1);}
|
||||
if(w)write(w,last);
|
||||
assert(SPIFFS.files.at("/contacts3")==next);
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")==next);
|
||||
assert(SPIFFS.exists("/contacts3.bak")==bool(completed<4));
|
||||
FakeFilesystem boot;boot.files=SPIFFS.files;assert(Writer::recover(&boot,"/contacts3",presence));assert(boot.files.at("/contacts3")==next);
|
||||
}
|
||||
reset(next);SPIFFS.files["/contacts3.bak"]=old;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,true);write(w,last);finish(w,true);}
|
||||
assert(SPIFFS.files.at("/contacts3")==last&&SPIFFS.files.at("/contacts3.bak")==next);
|
||||
#endif
|
||||
}
|
||||
void faultsAndCrash(){
|
||||
const auto old=image(304,1),next=image(608,2);
|
||||
for(bool deferred:{false,true})for(unsigned fault:{0U,1U,2U}) {
|
||||
reset(old);
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(w,true);write(w,next);
|
||||
if(fault)SPIFFS.fail_rename=fault;
|
||||
Progress result=Progress::Pending;unsigned n=0;
|
||||
while(result==Progress::Pending){assert(++n<2000);result=w.serviceCommit(true,8,deferred);}
|
||||
assert(result==(fault?Progress::Failed:Progress::Succeeded));
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")== (fault?old:next));
|
||||
const auto snapshots=SPIFFS.rename_snapshots;
|
||||
for(const auto& snapshot:snapshots){FakeFilesystem boot;boot.files=snapshot;assert(Writer::recover(&boot,"/contacts3",presence));const auto& bytes=boot.files.at("/contacts3");assert(bytes==old||bytes==next);}
|
||||
}
|
||||
}
|
||||
void counts(){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
for(bool deferred:{false,true})for(bool coalesce:{false,true}) {
|
||||
reset(image(152,9));const auto next=image(347*152,3);
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,coalesce);write(w,next);
|
||||
assert(finish(w,deferred)==108);assert(SPIFFS.files.at("/contacts3")==next);
|
||||
assert(SPIFFS.backend_writes==(coalesce?106U:211U));
|
||||
assert(SPIFFS.largest_backend_write==(coalesce?502U:251U));
|
||||
assert(SPIFFS.reads==825&&SPIFFS.backend_reads==104);
|
||||
}
|
||||
reset(image(152,9));Writer w(&SPIFFS,"/contacts3",presence);assert(w);write(w,image(347*152));
|
||||
assert(finish(w,false,1)==829&&SPIFFS.backend_writes==211&&SPIFFS.backend_reads==413);
|
||||
#endif
|
||||
}
|
||||
int main(int argc,char**argv){const std::string mode=argc>1?argv[1]:"all";if(mode=="contracts")contracts();else if(mode=="retire")retirement();else if(mode=="cancel")repeatedAndCancel();else if(mode=="faults")faultsAndCrash();else if(mode=="counts")counts();else assert(false);}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
// Executes the real CFT with the existing stdio-admission model.
|
||||
#include "FakeFilesystem.h"
|
||||
#define FILESYSTEM FakeFilesystem
|
||||
#include "transaction_under_test.h"
|
||||
#include <cstdio>
|
||||
|
||||
using Writer = mesh::ContactFileTransaction;
|
||||
using Progress = Writer::CommitProgress;
|
||||
using Bytes = std::vector<uint8_t>;
|
||||
Bytes image(size_t bytes) {
|
||||
Bytes result(bytes);
|
||||
for (size_t i = 0; i < bytes; ++i) result[i] = uint8_t(i * 23 + i / 7);
|
||||
return result;
|
||||
}
|
||||
bool presence(FakeFilesystem* fs, const char* path, bool& found) {
|
||||
++fs->metadata_probes;
|
||||
if (fs->metadata_error) return false;
|
||||
found = fs->exists(path);
|
||||
return true;
|
||||
}
|
||||
void begin(Writer& writer, bool background) {
|
||||
for (unsigned i = 0; !writer && i < 8; ++i)
|
||||
assert(writer.serviceBegin(background) != Writer::BeginProgress::Failed);
|
||||
assert(writer);
|
||||
}
|
||||
void records(Writer& writer, const Bytes& bytes, size_t cap) {
|
||||
assert(bytes.size() % 152 == 0);
|
||||
for (size_t offset = 0; offset < bytes.size(); offset += 152) {
|
||||
const unsigned before = SPIFFS.backend_writes;
|
||||
assert(writer.write(bytes.data() + offset, 152) == 152);
|
||||
assert(SPIFFS.backend_writes - before <= 1);
|
||||
assert(SPIFFS.largest_backend_write <= cap);
|
||||
}
|
||||
}
|
||||
Progress finish(Writer& writer, unsigned budget = 8) {
|
||||
Progress progress = Progress::Pending;
|
||||
for (unsigned i = 0; progress == Progress::Pending && i < 2000; ++i)
|
||||
progress = writer.serviceCommit(true, budget);
|
||||
assert(progress != Progress::Pending);
|
||||
return progress;
|
||||
}
|
||||
void measure(unsigned count, bool background) {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
const Bytes bytes = image(count * 152);
|
||||
Writer writer(&SPIFFS, "/contacts3", presence, true);
|
||||
begin(writer, background);
|
||||
const unsigned configurations = SPIFFS.buffer_config_calls;
|
||||
assert(writer.serviceBegin(!background) == Writer::BeginProgress::Ready);
|
||||
assert(SPIFFS.buffer_config_calls == configurations); // selection latched
|
||||
const size_t cap = background ? 502 : 251;
|
||||
records(writer, bytes, cap);
|
||||
assert(SPIFFS.writes == count);
|
||||
assert(SPIFFS.backend_writes == bytes.size() / cap);
|
||||
assert(finish(writer) == Progress::Succeeded);
|
||||
assert(SPIFFS.files.at("/contacts3") == bytes);
|
||||
assert(SPIFFS.backend_writes == (bytes.size() + cap - 1) / cap);
|
||||
assert(SPIFFS.largest_backend_write <= cap);
|
||||
assert(SPIFFS.missing_remove_logs == 0);
|
||||
printf("{\"records\":%u,\"background\":%s,\"bytes\":%zu,\"logical_writes\":%u,"
|
||||
"\"backend_writes\":%u,\"largest_backend_write\":%zu}\n",
|
||||
count, background ? "true" : "false", bytes.size(), SPIFFS.writes,
|
||||
SPIFFS.backend_writes, SPIFFS.largest_backend_write);
|
||||
}
|
||||
void synchronousDefault() {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
const Bytes bytes = image(347 * 152);
|
||||
Writer writer(&SPIFFS, "/contacts3", presence);
|
||||
assert(writer); records(writer, bytes, 251);
|
||||
assert(writer.commit());
|
||||
assert(SPIFFS.backend_writes == 211 && SPIFFS.largest_backend_write == 251);
|
||||
assert(SPIFFS.files.at("/contacts3") == bytes);
|
||||
}
|
||||
void configureTime(bool earlier, bool configure) {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
Writer writer(&SPIFFS, "/contacts3", presence, true);
|
||||
// Reach the open stream without configuring it, regardless of whether
|
||||
// background preparation also retires an earlier backup.
|
||||
unsigned passes = 0;
|
||||
while (SPIFFS.opens == 0) {
|
||||
assert(++passes < 16);
|
||||
assert(writer.serviceBegin(earlier) == Writer::BeginProgress::Pending);
|
||||
}
|
||||
assert(SPIFFS.buffer_config_calls == 0 && SPIFFS.writes == 0);
|
||||
assert(writer.serviceBegin(configure) == Writer::BeginProgress::Ready);
|
||||
const unsigned configurations = SPIFFS.buffer_config_calls;
|
||||
assert(writer.serviceBegin(!configure) == Writer::BeginProgress::Ready);
|
||||
assert(SPIFFS.buffer_config_calls == configurations);
|
||||
records(writer, image(4 * 152), configure ? 502 : 251);
|
||||
assert(SPIFFS.largest_backend_write == (configure ? 502 : 251));
|
||||
}
|
||||
void cancellation() {
|
||||
// gcd(152,502)=2:251 records are exactly76 full502 buffers.
|
||||
for (bool background : {false,true}) for (unsigned n : {1U,2U,3U,4U,250U,251U,252U,347U}) {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
const Bytes old = image(152), next = image(n * 152);
|
||||
SPIFFS.files["/contacts3"] = old;
|
||||
const size_t cap = background ? 502 : 251;
|
||||
unsigned before_close = 0;
|
||||
{
|
||||
Writer writer(&SPIFFS, "/contacts3", presence, true); begin(writer,background);
|
||||
records(writer,next,cap);
|
||||
assert(SPIFFS.backend_writes == next.size() / cap);
|
||||
before_close = SPIFFS.backend_writes;
|
||||
assert(SPIFFS.files.at("/contacts3") == old);
|
||||
}
|
||||
assert(SPIFFS.backend_writes-before_close == unsigned(next.size()%cap != 0));
|
||||
assert(SPIFFS.backend_writes == (next.size()+cap-1)/cap);
|
||||
assert(SPIFFS.largest_backend_write <= cap);
|
||||
assert(SPIFFS.files.at("/contacts3") == old && !SPIFFS.exists("/contacts3.tmp"));
|
||||
}
|
||||
}
|
||||
void setupFailures() {
|
||||
for (bool background : {false,true}) for (unsigned fault : {1U,2U,3U}) {
|
||||
SPIFFS=FakeFilesystem(); SPIFFS.emulate_stdio=true;
|
||||
const Bytes old=image(152); SPIFFS.files["/contacts3"]=old;
|
||||
if (fault==1) SPIFFS.fail_buffer_config=1;
|
||||
if (fault==2) SPIFFS.metadata_error=true;
|
||||
if (fault==3) SPIFFS.fail_create=SPIFFS.partial_create_failure=true;
|
||||
{
|
||||
Writer writer(&SPIFFS,"/contacts3",presence,true);
|
||||
auto result=Writer::BeginProgress::Pending;
|
||||
for (unsigned i=0;result==Writer::BeginProgress::Pending && i<8;++i)
|
||||
result=writer.serviceBegin(background);
|
||||
assert(result==Writer::BeginProgress::Failed && !writer);
|
||||
assert(writer.write(old.data(),old.size())==0);
|
||||
assert(writer.serviceCommit()==Progress::Failed);
|
||||
assert(SPIFFS.writes==0 && SPIFFS.backend_writes==0 && SPIFFS.reads==0);
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")==old);
|
||||
if(fault!=2) assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
}
|
||||
}
|
||||
void faultsAndCrash() {
|
||||
for (bool background : {false,true}) for(unsigned fault : {0U,1U,2U,3U,4U,5U}) {
|
||||
SPIFFS=FakeFilesystem(); SPIFFS.emulate_stdio=true;
|
||||
const Bytes old=image(152),next=image(4*152);SPIFFS.files["/contacts3"]=old;
|
||||
{
|
||||
Writer writer(&SPIFFS,"/contacts3",presence,true);begin(writer,background);
|
||||
if(fault==1) {
|
||||
SPIFFS.max_write=151;
|
||||
assert(writer.write(next.data(),152)==151);
|
||||
} else records(writer,next,background?502:251);
|
||||
if(fault==2) SPIFFS.fail_read="/contacts3.tmp";
|
||||
if(fault==3) SPIFFS.fail_rename=1;
|
||||
if(fault==4) SPIFFS.fail_rename=2;
|
||||
if(fault==5) {
|
||||
assert(writer.serviceCommit()==Progress::Pending); // flush before corrupting temp
|
||||
SPIFFS.files.at("/contacts3.tmp")[30]^=1;
|
||||
}
|
||||
const auto result=finish(writer);
|
||||
assert(result==(fault==0?Progress::Succeeded:Progress::Failed));
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")== (fault==0?next:old));
|
||||
assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
const auto snapshots=SPIFFS.rename_snapshots;
|
||||
for (const auto& snapshot:snapshots) {
|
||||
SPIFFS=FakeFilesystem();SPIFFS.files=snapshot;
|
||||
assert(Writer::recover(&SPIFFS,"/contacts3",presence));
|
||||
const auto& recovered=SPIFFS.files.at("/contacts3");
|
||||
assert(recovered==old || recovered==next);
|
||||
}
|
||||
}
|
||||
}
|
||||
int main(int argc,char** argv) {
|
||||
const std::string mode=argc>1?argv[1]:"all";
|
||||
if(mode=="measure") {
|
||||
for(unsigned n : {347U,349U,350U}) for(bool background : {false,true}) measure(n,background);
|
||||
} else if(mode=="default") synchronousDefault();
|
||||
else if(mode=="configure") {configureTime(true,false);configureTime(false,true);}
|
||||
else if(mode=="cancel") cancellation();
|
||||
else if(mode=="faults") {setupFailures();faultsAndCrash();}
|
||||
else assert(false && "unknown fixture mode");
|
||||
}
|
||||
@@ -91,10 +91,10 @@ class ContactCacheTest(unittest.TestCase):
|
||||
functions.append("bool DataStore::loadContactPages(")
|
||||
implementation += "\n".join(method(store, signature) for signature in functions)
|
||||
if disable_crc_batch:
|
||||
self.assertIn("_contact_write->serviceCommit(true, 8)", implementation)
|
||||
self.assertIn("_contact_write->serviceCommit(true, 8, true)", implementation)
|
||||
implementation = implementation.replace(
|
||||
"_contact_write->serviceCommit(true, 8)",
|
||||
"_contact_write->serviceCommit()")
|
||||
"_contact_write->serviceCommit(true, 8, true)",
|
||||
"_contact_write->serviceCommit(true, 1, true)")
|
||||
(temp / "store_under_test.h").write_text(implementation)
|
||||
packet = (ROOT / "src/Packet.cpp").read_text()
|
||||
(temp / "packet_under_test.h").write_text("namespace mesh {\n" + "\n".join(
|
||||
|
||||
@@ -9,7 +9,8 @@ import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SIGNATURE = "CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) {"
|
||||
SIGNATURE = ("CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1,\n"
|
||||
" bool defer_backup_cleanup = false) {")
|
||||
OPAQUE_FILE = """#pragma once
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
@@ -58,17 +59,29 @@ class ContactCRCBatchTest(unittest.TestCase):
|
||||
anchor = "ESP_VERIFY_BUFFER_SIZE = 128;"
|
||||
self.assertEqual(source.count(anchor), 1)
|
||||
source = source.replace(anchor, "ESP_VERIFY_BUFFER_SIZE = 4096;")
|
||||
elif negative == "background_stdio":
|
||||
anchor = "max_verify_chunks >= 8 ? 512 : ESP_VERIFY_BUFFER_SIZE"
|
||||
self.assertEqual(source.count(anchor), 1)
|
||||
source = source.replace(
|
||||
anchor, "max_verify_chunks >= 8 ? 4096 : ESP_VERIFY_BUFFER_SIZE")
|
||||
fixture = (ROOT / "test/fixtures/contact_crc_batch/test.cpp").read_text()
|
||||
if negative == "background_stdio":
|
||||
# Start with a large file so the negative executes a real >512B
|
||||
# refill, rather than failing only the selected-size assertion.
|
||||
self.assertEqual(fixture.count(" limitsAndTails();"), 1)
|
||||
fixture = fixture.replace(" limitsAndTails();", " verify(347 * 152, 8);")
|
||||
# Count actual commit() calls without replacing its body or algorithm.
|
||||
source = source.replace(SIGNATURE, SIGNATURE + "\n ++fixture_commit_calls;")
|
||||
with tempfile.TemporaryDirectory(prefix="mesh-contact-crc-") as directory:
|
||||
temp = Path(directory)
|
||||
(temp / "transaction_under_test.h").write_text(source)
|
||||
(temp / "fixture.cpp").write_text(fixture)
|
||||
binary = temp / "fixture"
|
||||
result = subprocess.run([
|
||||
"c++", "-std=c++17", "-O1", "-g", "-Wall", "-Wextra",
|
||||
"-fsanitize=address,undefined", "-fno-omit-frame-pointer",
|
||||
"-D" + platform + "=1", "-I", str(temp), "-I", str(ROOT / "src"),
|
||||
str(ROOT / "test/fixtures/contact_crc_batch/test.cpp"),
|
||||
str(temp / "fixture.cpp"),
|
||||
"-o", str(binary),
|
||||
], capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
@@ -76,7 +89,8 @@ class ContactCRCBatchTest(unittest.TestCase):
|
||||
if negative:
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
markers = {"clamp": "reads <= bound", "synchronous": "fixture_commit_calls == 829",
|
||||
"stdio": "largest_backend_read <= 128"}
|
||||
"stdio": "reader_buffer_size == selected",
|
||||
"background_stdio": "backend_bytes <= 512"}
|
||||
self.assertIn(markers[negative], result.stderr)
|
||||
return
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
@@ -85,8 +99,10 @@ class ContactCRCBatchTest(unittest.TestCase):
|
||||
def test_esp_bounds_tails_failures_crc_cancellation_and_crash(self):
|
||||
result = self.run_native()
|
||||
self.assertEqual(result, {"one_passes": 829, "eight_passes": 108,
|
||||
"logical_reads": 825, "backend_reads": 413,
|
||||
"max_pass_reads": 8, "max_pass_refills": 4})
|
||||
"logical_reads": 825, "backend_reads": 104,
|
||||
"max_pass_reads": 8, "max_pass_refills": 1,
|
||||
"max_pass_backend_bytes": 512, "reader_buffer_size": 512,
|
||||
"one_backend_reads": 413, "one_reader_buffer_size": 128})
|
||||
|
||||
def test_other_platform_ignores_requested_batch(self):
|
||||
result = self.run_native("RP2040_PLATFORM")
|
||||
@@ -103,6 +119,9 @@ class ContactCRCBatchTest(unittest.TestCase):
|
||||
def test_large_stdio_read_ahead_is_detected(self):
|
||||
self.run_native(negative="stdio")
|
||||
|
||||
def test_large_background_stdio_read_ahead_is_detected(self):
|
||||
self.run_native(negative="background_stdio")
|
||||
|
||||
def test_opaque_xtensa_fixture_frame_and_object_sizes(self):
|
||||
compiler = os.environ.get("CONTACT_CRC_XTENSA_CXX") or shutil.which("xtensa-esp32s3-elf-g++")
|
||||
if not compiler:
|
||||
@@ -134,7 +153,7 @@ class ContactCRCBatchTest(unittest.TestCase):
|
||||
frames = [line.split("\t") for line in (temp / "frame.su").read_text().splitlines()
|
||||
if "ContactFileTransaction::serviceCommit(" in line]
|
||||
self.assertEqual(len(frames), 1)
|
||||
self.assertEqual(int(frames[0][1]), 112)
|
||||
self.assertEqual(int(frames[0][1]), 128)
|
||||
nm = str(Path(compiler).with_name("xtensa-esp32s3-elf-nm"))
|
||||
symbols = subprocess.check_output([nm, "-S", str(temp / "frame.o")], text=True)
|
||||
size = next(int(line.split()[1], 16) for line in symbols.splitlines()
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import json,subprocess,tempfile,unittest
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
FIXTURE=ROOT/"test/fixtures/contact_publication/test.cpp"
|
||||
class Publication(unittest.TestCase):
|
||||
def run_native(self,mode,platform='ESP32_PLATFORM',negative=None):
|
||||
source=(ROOT/'src/helpers/ContactFileTransaction.h').read_text().replace('#include "IdentityStore.h"','// native supplied File').replace('#include "PersistentStoreFormat.h"','#include <helpers/PersistentStoreFormat.h>')
|
||||
if negative=='cleanup':
|
||||
a='if (!defer_backup_cleanup)';self.assertEqual(source.count(a),1);source=source.replace(a,'if (true)')
|
||||
elif negative=='retirement':
|
||||
a='ok = _fs->remove(_backup);';self.assertEqual(source.count(a),2);source=source.replace(a,'ok = true; // negative: leave previous backup',1)
|
||||
with tempfile.TemporaryDirectory(prefix='mesh-publication-') as folder:
|
||||
p=Path(folder);(p/'transaction_under_test.h').write_text(source);binary=p/'test'
|
||||
result=subprocess.run(['c++','-std=c++17','-O1','-g','-Wall','-Wextra','-fsanitize=address,undefined','-fno-omit-frame-pointer','-D'+platform+'=1','-I',str(p),'-I',str(ROOT/'src'),'-I',str(ROOT/'test/fixtures/contact_cache'),'-I',str(ROOT/'test/fixtures/contact_cache/mocks'),str(FIXTURE),'-o',str(binary)],text=True,capture_output=True)
|
||||
self.assertEqual(result.returncode,0,result.stderr)
|
||||
result=subprocess.run([str(binary),mode],text=True,capture_output=True)
|
||||
if negative:
|
||||
self.assertNotEqual(result.returncode,0,result.stdout)
|
||||
self.assertIn('SPIFFS.',result.stderr)
|
||||
else:self.assertEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
def test_default_and_deferred_success_contracts(self):self.run_native('contracts')
|
||||
def test_retirement_and_recovery_failure_before_temp_ownership(self):self.run_native('retire')
|
||||
def test_repeated_transaction_all_begin_cancellation_boundaries(self):self.run_native('cancel')
|
||||
def test_publish_failures_and_every_rename_crash(self):self.run_native('faults')
|
||||
def test_251_502_counts_and_unchanged_commit_cadence(self):self.run_native('counts')
|
||||
def test_other_platform_retains_cleanup_despite_true_arg(self):self.run_native('contracts','RP2040_PLATFORM')
|
||||
def test_removed_defer_guard_is_detected(self):self.run_native('contracts',negative='cleanup')
|
||||
def test_omitted_retirement_is_detected(self):self.run_native('retire',negative='retirement')
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Execute background502/default251 admission bounds on the real CFT."""
|
||||
from pathlib import Path
|
||||
import json,os,subprocess,tempfile,unittest
|
||||
ROOT=Path(os.environ.get("CONTACT_WRITER_ROOT",Path(__file__).resolve().parents[1]))
|
||||
HEADER=Path(os.environ.get("CONTACT_WRITER_HEADER",ROOT/"src/helpers/ContactFileTransaction.h"))
|
||||
DATASTORE=Path(os.environ.get("CONTACT_WRITER_DATASTORE",ROOT/"examples/companion_radio/DataStore.cpp"))
|
||||
FIXTURE=Path(__file__).resolve().parent/"fixtures/contact_writer_batch/test.cpp"
|
||||
class WriterBatch(unittest.TestCase):
|
||||
def run_native(self,mode,negative=None):
|
||||
source=HEADER.read_text().replace('#include "IdentityStore.h"','// native supplied FILESYSTEM').replace(
|
||||
'#include "PersistentStoreFormat.h"','#include <helpers/PersistentStoreFormat.h>')
|
||||
if negative=="default":
|
||||
anchor='while (serviceBegin() == BeginProgress::Pending) {}'
|
||||
self.assertEqual(source.count(anchor),1)
|
||||
source=source.replace(anchor,'while (serviceBegin(true) == BeginProgress::Pending) {}')
|
||||
if negative=="large":
|
||||
anchor='ESP_BACKGROUND_WRITE_BUFFER_SIZE = 502;'
|
||||
self.assertEqual(source.count(anchor),1);source=source.replace(anchor,'ESP_BACKGROUND_WRITE_BUFFER_SIZE = 4096;')
|
||||
# Reuse the existing immediate-full-newlib stdio model unchanged.
|
||||
with tempfile.TemporaryDirectory(prefix='mesh-writer-batch-') as directory:
|
||||
tmp=Path(directory);(tmp/'transaction_under_test.h').write_text(source)
|
||||
binary=tmp/'fixture'
|
||||
cmd=['c++','-std=c++17','-O1','-g','-Wall','-Wextra','-fsanitize=address,undefined',
|
||||
'-fno-omit-frame-pointer','-DESP32_PLATFORM=1','-I',str(tmp),'-I',str(ROOT/'src'),
|
||||
'-I',str(ROOT/'test/fixtures/contact_cache'),'-I',str(ROOT/'test/fixtures/contact_cache/mocks'),
|
||||
str(FIXTURE),'-o',str(binary)]
|
||||
compiled=subprocess.run(cmd,text=True,capture_output=True)
|
||||
self.assertEqual(compiled.returncode,0,compiled.stderr)
|
||||
result=subprocess.run([str(binary),mode],text=True,capture_output=True)
|
||||
if negative:
|
||||
self.assertNotEqual(result.returncode,0,result.stdout)
|
||||
self.assertIn('largest_backend_write <= cap',result.stderr)
|
||||
return
|
||||
self.assertEqual(result.returncode,0,result.stdout+result.stderr)
|
||||
return [json.loads(line) for line in result.stdout.splitlines()]
|
||||
def test_counts_admissions_and_exact_committed_bytes(self):
|
||||
rows=self.run_native('measure');self.assertEqual(len(rows),6)
|
||||
for row in rows:
|
||||
n=row['records'];cap=502 if row['background'] else 251
|
||||
self.assertEqual(row['logical_writes'],n)
|
||||
self.assertEqual(row['bytes'],n*152)
|
||||
self.assertEqual(row['backend_writes'],(n*152+cap-1)//cap)
|
||||
self.assertEqual(row['largest_backend_write'],cap)
|
||||
self.assertEqual([(r['backend_writes']) for r in rows],[211,106,212,106,212,106])
|
||||
def test_synchronous_constructor_keeps_default251(self):self.run_native('default')
|
||||
def test_configure_time_selection_is_latched(self):self.run_native('configure')
|
||||
def test_cancellation_tail_and_old_target(self):self.run_native('cancel')
|
||||
def test_fail_closed_config_short_write_corruption_and_crash(self):self.run_native('faults')
|
||||
def test_old_default_contract_regression_is_detected(self):self.run_native('default','default')
|
||||
def test_4096_burst_is_detected(self):self.run_native('measure','large')
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in New Issue
Block a user