Reduce contact RAM across boards and compact Full firmware

.github/workflows/run-unit-tests.yml

@@ -459,6 +459,7 @@ jobs:
           python3 -B test/test_companion_hardware_serial.py -v
           python3 -B test/test_companion_hardware_serial_gps.py -v
           python3 -B test/test_companion_contact_lifetime.py -v
+          python3 -B test/test_contact_indices.py -v
           python3 -B test/test_companion_identity_cli.py -v
           python3 -B test/test_companion_packet_stream.py -v
           python3 -B test/test_companion_uncached_storage.py -v

src/helpers/BaseChatMesh.cpp

@@ -38,8 +38,8 @@ bool BaseChatMesh::initializeContactStorage() {
   ContactInfo* expanded_contacts = static_cast<ContactInfo*>(
       heap_caps_calloc(requested_capacity, sizeof(ContactInfo),
                        MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
-  int* expanded_sort_array = static_cast<int*>(
-      heap_caps_calloc(requested_capacity, sizeof(int),
+  ContactIndex* expanded_sort_array = static_cast<ContactIndex*>(
+      heap_caps_calloc(requested_capacity, sizeof(ContactIndex),
                        MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
   if (expanded_contacts == NULL || expanded_sort_array == NULL) {
     if (expanded_contacts != NULL) heap_caps_free(expanded_contacts);
@@ -1056,8 +1056,8 @@ void BaseChatMesh::resetPathTo(ContactInfo& recipient) {
 static ContactInfo* table;  // pass via global :-(
 
 static int cmp_contact_list_order(const void *a, const void *b) {
-  int a_idx = *((int *)a);
-  int b_idx = *((int *)b);
+  int a_idx = *static_cast<const ContactIndex*>(a);
+  int b_idx = *static_cast<const ContactIndex*>(b);
   return mesh::compareContactListOrder(
       table[a_idx].flags, table[a_idx].last_advert_timestamp,
       table[b_idx].flags, table[b_idx].last_advert_timestamp);

src/helpers/BaseChatMesh.h

@@ -47,6 +47,14 @@ public:
 
 #define MAX_ANON_CONTACTS  8
 
+// These arrays hold contact-table positions, never negative sentinels. Keep
+// wider indices for configurations whose complete table exceeds 16 bits.
+#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535
+using ContactIndex = uint16_t;
+#else
+using ContactIndex = int;
+#endif
+
 #ifndef MAX_CONNECTIONS
   #define MAX_CONNECTIONS  16
 #endif
@@ -77,16 +85,16 @@ class BaseChatMesh : public mesh::Mesh {
   };
   ContactInfo* contacts;
   ContactInfo contacts_fallback[CONTACT_PSRAM_FALLBACK_TOTAL_SLOTS];
-  int* sort_array;
-  int sort_array_fallback[CONTACT_PSRAM_FALLBACK_TOTAL_SLOTS];
+  ContactIndex* sort_array;
+  ContactIndex sort_array_fallback[CONTACT_PSRAM_FALLBACK_TOTAL_SLOTS];
   int contact_capacity;
 #else
   ContactInfo contacts[MAX_CONTACTS+MAX_ANON_CONTACTS];
-  int sort_array[MAX_CONTACTS+MAX_ANON_CONTACTS];
+  ContactIndex sort_array[MAX_CONTACTS+MAX_ANON_CONTACTS];
 #endif
   int num_contacts = 0;
   uint32_t contact_table_revision;
-  int matching_peer_indexes[MAX_SEARCH_RESULTS];
+  ContactIndex matching_peer_indexes[MAX_SEARCH_RESULTS];
   unsigned long txt_send_timeout;
 #ifdef MAX_GROUP_CHANNELS
   ChannelDetails channels[MAX_GROUP_CHANNELS];

src/helpers/BeaconManager.cpp

@@ -5,6 +5,7 @@
 #include <Mesh.h>
 #include <helpers/FilePresence.h>
 #include <helpers/MeshtextChat.h>
+#include <helpers/PersistentStoreFormat.h>
 #include <helpers/UTF8Helpers.h>
 #include <Arduino.h>
 #include <ctype.h>
@@ -27,14 +28,7 @@ const char* const IdPath = "/beacon_ids";
 const char* const IdBackup = "/beacon_ids.bak";
 const char* const IdTemp = "/beacon_ids.tmp";
 
-uint32_t crcUpdate(uint32_t crc, const uint8_t* bytes, size_t size) {
-  while (size--) {
-    crc ^= *bytes++;
-    for (unsigned i = 0; i < 8; ++i)
-      crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320U : 0);
-  }
-  return crc;
-}
+using storage::updateCRC32;
 void put32(uint8_t* bytes, uint32_t value) {
   for (unsigned i = 0; i < 4; ++i) bytes[i] = uint8_t(value >> (8 * i));
 }
@@ -230,14 +224,14 @@ BeaconManager::ReadResult BeaconManager::readConfig(
   if (header[4] != 1) { file.close(); return ReadResult::Future; }
   if (file.size() != ConfigSize || header[5] != SlotCount || header[6] > 1
       || !validInterval(header[7])) { file.close(); return ReadResult::Invalid; }
-  uint32_t crc = crcUpdate(0xffffffffU, header, HeaderSize);
+  uint32_t crc = updateCRC32(0xffffffffU, header, HeaderSize);
   bool valid = true;
   for (unsigned i = 0; i < SlotCount; ++i) {
     uint8_t bytes[RecordSize];
     if (file.read(bytes, sizeof(bytes)) != int(sizeof(bytes))) {
       file.close(); return ReadResult::Unreadable;
     }
-    crc = crcUpdate(crc, bytes, sizeof(bytes));
+    crc = updateCRC32(crc, bytes, sizeof(bytes));
     BeaconSlot slot;
     // Disk validity is independent of the current board's radio capabilities.
     // Preserve unsupported slots so an upgrade can still query/repair them.
@@ -269,7 +263,7 @@ BeaconManager::ReadResult BeaconManager::readIds(const char* path, uint32_t& las
   if (memcmp(bytes, "BCI!", 4)) return ReadResult::Invalid;
   if (bytes[4] != 1) return ReadResult::Future;
   if (bytes[5] || bytes[6] || bytes[7]
-      || get32(bytes + 12) != crcUpdate(0xffffffffU, bytes, 12)) return ReadResult::Invalid;
+      || get32(bytes + 12) != updateCRC32(0xffffffffU, bytes, 12)) return ReadResult::Invalid;
   last = get32(bytes + 8);
   return ReadResult::Valid;
 }
@@ -313,14 +307,14 @@ bool BeaconManager::saveConfig(unsigned changed_slot, const BeaconSlot* changed,
   if (!openWrite(fs_, ConfigTemp, file)) return false;
   uint8_t header[HeaderSize] = {'B', 'C', 'N', '!', 1, SlotCount,
                                 uint8_t(initialized), hours};
-  uint32_t crc = crcUpdate(0xffffffffU, header, sizeof(header));
+  uint32_t crc = updateCRC32(0xffffffffU, header, sizeof(header));
   bool ok = file.write(header, sizeof(header)) == sizeof(header);
   for (unsigned i = 0; i < SlotCount && ok; ++i) {
     BeaconSlot empty;
     const BeaconSlot& slot = clear_all ? empty
         : (changed && changed_slot == i ? *changed : (slots_ ? slots_[i] : empty));
     uint8_t bytes[RecordSize]; encodeSlot(slot, bytes);
-    crc = crcUpdate(crc, bytes, sizeof(bytes));
+    crc = updateCRC32(crc, bytes, sizeof(bytes));
     ok = file.write(bytes, sizeof(bytes)) == sizeof(bytes);
   }
   uint8_t trailer[4]; put32(trailer, crc);
@@ -340,7 +334,7 @@ bool BeaconManager::saveIds(uint32_t last) {
   File file = makeFile(fs_);
   if (!openWrite(fs_, IdTemp, file)) { ids_hold_ = true; return false; }
   uint8_t bytes[IdSize] = {'B', 'C', 'I', '!', 1, 0, 0, 0};
-  put32(bytes + 8, last); put32(bytes + 12, crcUpdate(0xffffffffU, bytes, 12));
+  put32(bytes + 8, last); put32(bytes + 12, updateCRC32(0xffffffffU, bytes, 12));
   bool ok = file.write(bytes, sizeof(bytes)) == sizeof(bytes);
   file.flush(); file.close();
   uint32_t verified = 0;

src/helpers/ContactInfo.h

@@ -25,19 +25,19 @@ struct ContactInfo {
   mesh::ContactPathRef path_ref;
 #else
   uint8_t out_path[MAX_PATH_SIZE];
+#endif
+  // Runtime-only position in the paged companion contact store.  It is not
+  // part of the on-disk record; the record's page/slot supplies it on load.
+  // Mutable keeps persistence bookkeeping out of contact protocol semantics.
+  // Beside the cached path handle, it fills the alignment gap before timestamps.
+#if defined(NRF52_PLATFORM)
+  mutable uint16_t storage_slot = mesh::storage::CONTACT_SLOT_NONE;
 #endif
   uint32_t last_advert_timestamp = 0;   // by THEIR clock
   uint32_t lastmod = 0;  // by OUR clock
   int32_t gps_lat = 0, gps_lon = 0;    // 6 dec places
   uint32_t sync_since = 0;
 
-  // Runtime-only position in the paged companion contact store.  It is not
-  // part of the on-disk record; the record's page/slot supplies it on load.
-  // Mutable keeps persistence bookkeeping out of contact protocol semantics.
-#if defined(NRF52_PLATFORM)
-  mutable uint16_t storage_slot = mesh::storage::CONTACT_SLOT_NONE;
-#endif
-
   ContactInfo() : shared_secret_valid(false), tx_radio(mesh::RADIO_TX_AUTO) {}
 
   // The pointer is a short-lived borrow. Copy bytes for a queued operation;

src/helpers/ContactPathCache.h

@@ -3,6 +3,7 @@
 #include <stdint.h>
 #include <stddef.h>
 #include <string.h>
+#include "PersistentStoreFormat.h"
 
 namespace mesh {
 
@@ -95,13 +96,7 @@ class ContactPathPool : public ContactPathStorage {
   bool _flushing = false;
 
   static uint32_t checksum(const uint8_t path[64]) {
-    uint32_t crc = 0xffffffff;
-    for (size_t i = 0; i < 64; ++i) {
-      crc ^= path[i];
-      for (unsigned bit = 0; bit < 8; ++bit)
-        crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320UL : 0);
-    }
-    return crc;
+    return storage::updateCRC32(0xffffffffUL, path, 64);
   }
 
   Slot* slot(uint16_t h) {

src/helpers/MorseSettings.h

@@ -2,6 +2,7 @@
 
 #include <helpers/IdentityStore.h>
 #include <helpers/MorseTx.h>
+#include <helpers/PersistentStoreFormat.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -27,13 +28,7 @@ inline bool validMorseSettings(const MorseSettings& prefs) {
 }
 
 inline uint32_t morseSettingsCRC(const uint8_t* data, size_t size) {
-  uint32_t crc = 0xffffffffUL;
-  while (size--) {
-    crc ^= *data++;
-    for (uint8_t bit = 0; bit < 8; ++bit)
-      crc = (crc >> 1) ^ ((crc & 1U) ? 0xedb88320UL : 0);
-  }
-  return crc;
+  return storage::updateCRC32(0xffffffffUL, data, size);
 }
 
 inline void encodeMorseSettings(const MorseSettings& prefs,

src/helpers/RadioProfileCLI.cpp

@@ -5,6 +5,7 @@
 #endif
 #include "RadioProfileCommandUtils.h"
 #include "FilePresence.h"
+#include "PersistentStoreFormat.h"
 #include "radiolib/RXPowerSaving.h"
 #include "radiolib/HilTxFrequencyGuard.h"
 #include <Arduino.h>
@@ -21,12 +22,7 @@ const char* const TempPath = "/radio_profiles.tmp";
 // acknowledgement leaves the previous saved radio untouched.
 const char* const ReplyPath = "/radio_profiles.reply";
 uint32_t checksum(const uint8_t* p, size_t n) {
-  uint32_t crc = 0xffffffffU;
-  while (n--) {
-    crc ^= *p++;
-    for (int i = 0; i < 8; ++i) crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320U : 0);
-  }
-  return crc;
+  return storage::updateCRC32(0xffffffffU, p, n);
 }
 const char* modeName(RadioProfileMode m) {
   return m == RadioProfileMode::RxTx ? "rxtx" : m == RadioProfileMode::Rx ? "rx" : "off";

src/helpers/bible/ReaderLookup.h

@@ -15,21 +15,25 @@ static constexpr uint8_t kChapterVerses[] = {
 
 struct Reference { uint8_t chapter; uint8_t verse; };
 enum class ParseResult { NoMatch, Invalid, Valid };
-struct BlockIndex {
-  uint32_t offset;
+template <typename Offset> struct BlockIndexT {
+  Offset offset;
   uint16_t packed;
   uint16_t plain;
   uint16_t first_verse;
 };
-struct Corpus {
+template <typename Offset> struct CorpusT {
   const char* translation;
   const char* attribution;
   const uint8_t* data;
   uint32_t data_size;
-  const BlockIndex* blocks;
+  const BlockIndexT<Offset>* blocks;
   uint16_t block_count;
   uint16_t verse_count;
 };
+// Keep the original general-purpose types available. Generated corpora select
+// a narrower offset only when every block start fits, without a runtime branch.
+using BlockIndex = BlockIndexT<uint32_t>;
+using Corpus = CorpusT<uint32_t>;
 enum class LookupResult { Found, Missing, Invalid, Corrupt };
 
 inline bool space(char c) { return c == ' ' || c == '\t'; }
@@ -84,7 +88,8 @@ inline ParseResult parse(const char* command, Reference& ref) {
 
 // One independently compressed block is decoded on demand. Const tables/data
 // stay in memory-mapped flash on both target families; there is no heap/cache.
-inline LookupResult lookup(const Corpus& corpus, Reference ref,
+template <typename Offset>
+inline LookupResult lookup(const CorpusT<Offset>& corpus, Reference ref,
                            char* scratch, size_t capacity, const char*& text) {
   text = nullptr;
   uint16_t index;
@@ -97,7 +102,7 @@ inline LookupResult lookup(const Corpus& corpus, Reference ref,
   unsigned block_index = 0;
   while (block_index + 1 < corpus.block_count &&
          corpus.blocks[block_index + 1].first_verse <= index) ++block_index;
-  const BlockIndex block = corpus.blocks[block_index];
+  const BlockIndexT<Offset> block = corpus.blocks[block_index];
   if (block.plain == 0 || block.plain > kBlockSize || block.packed == 0 ||
       block.offset > corpus.data_size || block.packed > corpus.data_size - block.offset ||
       block.first_verse > index) return LookupResult::Corrupt;

test/fixtures/companion_contact_lifetime/test.cpp

@@ -92,9 +92,9 @@ class BaseChatMesh {
 public:
 #if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
   ContactInfo contacts_fallback[MAX_ANON_CONTACTS + 16];
-  int sort_array_fallback[MAX_ANON_CONTACTS + 16] = {};
+  ContactIndex sort_array_fallback[MAX_ANON_CONTACTS + 16] = {};
   ContactInfo* contacts = contacts_fallback;
-  int* sort_array = sort_array_fallback;
+  ContactIndex* sort_array = sort_array_fallback;
   int contact_capacity = MAX_ANON_CONTACTS + 16;
 #else
   ContactInfo contacts[MAX_ANON_CONTACTS + MAX_CONTACTS];

test/fixtures/companion_reader/test_reader_lookup.cpp

@@ -8,6 +8,22 @@
 
 using namespace mesh::bible;
 
+template <typename Offset>
+static void checkOffsetWidth(uint32_t offset) {
+  static const uint8_t stored[] = {0x01, 0x02, 0x00, 0xfd, 0xff, 'x', 0x00};
+  std::vector<uint8_t> data(offset + sizeof(stored));
+  memcpy(data.data() + offset, stored, sizeof(stored));
+  const BlockIndexT<Offset> index = {static_cast<Offset>(offset), sizeof(stored), 2, 0};
+  CorpusT<Offset> corpus = {"test", "test", data.data(), uint32_t(data.size()), &index, 1, kVerseCount};
+  char scratch[kBlockSize];
+  const char* text;
+  assert(lookup(corpus, {1, 1}, scratch, sizeof(scratch), text) == LookupResult::Found);
+  assert(strcmp(text, "x") == 0);
+  corpus.data_size--;
+  assert(lookup(corpus, {1, 1}, scratch, sizeof(scratch), text) == LookupResult::Corrupt);
+  assert(text == nullptr);
+}
+
 static void drain(Stream& output) {
   unsigned calls = 0;
   while (mesh::readerCommandActive()) {
@@ -70,6 +86,10 @@ static void checkSelection(unsigned chapter, unsigned first, unsigned last, bool
 }
 
 int main(int argc, char**) {
+  static_assert(sizeof(BlockIndexT<uint16_t>) == 8, "halfword index alignment");
+  static_assert(sizeof(BlockIndexT<uint32_t>) == 12, "wide index alignment");
+  checkOffsetWidth<uint16_t>(UINT16_MAX);
+  checkOffsetWidth<uint32_t>(uint32_t(UINT16_MAX) + 1);
   Reference ref = {};
   for (const char* cmd : {"get reader 3:16", " GET\treader 3:16 \t", "get reader 03:16"}) {
     assert(parse(cmd, ref) == ParseResult::Valid && ref.chapter == 3 && ref.verse == 16);
@@ -87,7 +107,7 @@ int main(int argc, char**) {
     assert(!mesh::handleReaderCommand(cmd, output) && output.text.empty());
   }
   assert(parse(nullptr, ref) == ParseResult::NoMatch);
-  const Corpus& corpus = generated::readerCorpus;
+  const auto& corpus = generated::readerCorpus;
   struct Guarded { uint32_t pre; char scratch[kBlockSize]; uint32_t post; } buf = {};
   buf.pre = 0x11223344; buf.post = 0xaabbccdd;
   unsigned count = 0;
@@ -189,7 +209,7 @@ int main(int argc, char**) {
   const char* text;
   assert(lookup(corpus, {22, 1}, buf.scratch, kBlockSize, text) == LookupResult::Invalid);
   assert(lookup(corpus, {1, 1}, buf.scratch, kBlockSize - 1, text) == LookupResult::Corrupt);
-  Corpus bad = corpus;
+  auto bad = corpus;
   bad.data_size = 0;
   assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
   bad = corpus; bad.block_count = 0;
@@ -200,7 +220,7 @@ int main(int argc, char**) {
   // One-byte stored DEFLATE block: distinguish an absent verse marker from
   // damaged text without a terminator, and never return an unsafe pointer.
   uint8_t stored[] = {0x01, 0x01, 0x00, 0xfe, 0xff, 0x00};
-  const BlockIndex stored_index = {0, sizeof(stored), 1, 0};
+  const generated::ReaderBlockIndex stored_index = {0, sizeof(stored), 1, 0};
   bad = corpus; bad.data = stored; bad.data_size = sizeof(stored);
   bad.blocks = &stored_index; bad.block_count = 1;
   assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Missing);
@@ -209,7 +229,7 @@ int main(int argc, char**) {
   assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
   assert(text == nullptr);
   assert(lookup(bad, {1, 2}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);
-  std::vector<BlockIndex> blocks(corpus.blocks, corpus.blocks + corpus.block_count);
+  std::vector<generated::ReaderBlockIndex> blocks(corpus.blocks, corpus.blocks + corpus.block_count);
   bad = corpus; bad.blocks = blocks.data();
   for (int mutation = 0; mutation != 5; ++mutation) {
     blocks[0] = corpus.blocks[0];
@@ -217,7 +237,7 @@ int main(int argc, char**) {
       case 0: blocks[0].plain = kBlockSize + 1; break;
       case 1: blocks[0].packed--; break;
       case 2: blocks[0].packed++; break; // Trailing bytes are rejected.
-      case 3: blocks[0].offset = UINT32_MAX; break;
+      case 3: blocks[0].offset = static_cast<decltype(blocks[0].offset)>(-1); break;
       case 4: blocks[0].first_verse = 1; break;
     }
     assert(lookup(bad, {1, 1}, buf.scratch, kBlockSize, text) == LookupResult::Corrupt);

test/fixtures/contact_cache/test_contact_cache.cpp

@@ -500,7 +500,7 @@ static void raw_read_recheck_preserves_old_and_new_unread_page_scopes() {
 
 static void routes_survive_eviction_and_full_sync() {
   Fixture f;
-  assert(sizeof(ContactInfo) <= 100); // all 350 records retain compact handles
+  assert(sizeof(ContactInfo) <= 92); // no per-contact padding tax across all 350 cached records
   const unsigned reads = SPIFFS.reads;
   f.check(0, route(0));
   assert(SPIFFS.reads == reads + 1);

test/test_companion_contact_lifetime.py

@@ -101,7 +101,9 @@ class CompanionContactLifetimeTests(unittest.TestCase):
             work = Path(directory)
             (work / "production_constants.inc").write_text(constants, encoding="ascii")
             (work / "production_types.inc").write_text(
-                "namespace mesh {\n" + extract_braced((ROOT / "src/Mesh.h").read_text(), "class GroupChannel")
+                base_header[base_header.index("#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535"):
+                            base_header.index("#ifndef MAX_CONNECTIONS")]
+                + "namespace mesh {\n" + extract_braced((ROOT / "src/Mesh.h").read_text(), "class GroupChannel")
                 + ";\n}\n" + extract_braced((ROOT / "src/helpers/ChannelDetails.h").read_text(), "struct ChannelDetails")
                 + ";\n" + extract_braced(base_header, "struct ConnectionInfo") + ";\n", encoding="ascii")
             (work / "production_entry.inc").write_text(

test/test_companion_reader.py

@@ -8,6 +8,7 @@ import shutil
 import subprocess
 import tempfile
 import unittest
+from unittest.mock import patch
 import zlib
 
 from test_message_navigation import PAIRING_PREAMBLE as PREAMBLE
@@ -126,7 +127,23 @@ class CompanionReaderTest(unittest.TestCase):
         packed, blocks = packer.pack(self.document)
         self.assertEqual(sum(block[2] for block in blocks), 96568)
         self.assertEqual(len(blocks), 24)
-        self.assertEqual(len(packed) + 12 * len(blocks), 38654)
+        self.assertEqual(len(packed), 38293)
+        self.assertEqual(packer.block_index_layout(blocks), ("uint16_t", 8))
+        self.assertEqual(len(packed) + 8 * len(blocks), 38485)
+
+    def test_index_offsets_widen_before_any_block_start_can_truncate(self):
+        self.assertEqual(packer.block_index_layout([(65535, 7, 2, 0)]), ("uint16_t", 8))
+        self.assertEqual(packer.block_index_layout([(65536, 7, 2, 0)]), ("uint32_t", 12))
+        self.assertEqual(packer.block_index_layout([(0xFFFFFFFF, 7, 2, 0)]), ("uint32_t", 12))
+        for offset in (-1, 0x100000000):
+            with self.assertRaises(ValueError):
+                packer.block_index_layout([(offset, 7, 2, 0)])
+        with patch.object(packer, "pack", return_value=(b"", [(65536, 7, 2, 0)])):
+            header = packer.render(self.document)
+        self.assertIn("using ReaderBlockIndex = BlockIndexT<uint32_t>;", header)
+        self.assertIn("using ReaderCorpus = CorpusT<uint32_t>;", header)
+        self.assertIn("sizeof(ReaderBlockIndex) == 12", header)
+        self.assertIn("{65536, 7, 2, 0}", header)
 
     def test_invalid_sources_are_rejected(self):
         for bad_text in ("", "leading\ncommand", "bad\x1b[2J", "bad\0text",

test/test_contact_indices.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""Execute production contact indexing at compact/wide and PSRAM boundaries."""
+from pathlib import Path
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from test_replay_reset_integration import extract_braced
+
+ROOT = Path(__file__).resolve().parents[1]
+
+HARNESS = r'''
+#include <cassert>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <algorithm>
+#include <memory>
+#include <new>
+#include <set>
+#include <type_traits>
+#include <utility>
+#include <vector>
+#include <helpers/ContactListOrder.h>
+#define MESH_DEBUG_PRINTLN(...) ((void)0)
+@INDEX_TYPES@
+struct ContactInfo {
+  struct Identity {
+    uint8_t prefix = 0;
+    uint32_t ordinal = 0;
+    bool isHashMatch(const uint8_t* hash) const { return prefix == *hash; }
+  } id;
+  uint8_t flags = 0;
+  uint32_t last_advert_timestamp = 0;
+};
+@VISITOR@
+static unsigned allocations = 0, fail_allocation = 0;
+static std::set<void*> outstanding;
+static std::vector<std::pair<size_t, size_t>> allocation_sizes;
+static constexpr int MALLOC_CAP_SPIRAM = 1, MALLOC_CAP_8BIT = 2;
+static void* heap_caps_calloc(size_t count, size_t size, int) {
+  allocation_sizes.push_back({count, size});
+  if (++allocations == fail_allocation) return nullptr;
+  void* pointer = calloc(count, size);
+  assert(pointer != nullptr);
+  outstanding.insert(pointer);
+  return pointer;
+}
+static void heap_caps_free(void* pointer) {
+  assert(outstanding.erase(pointer) == 1);
+  free(pointer);
+}
+class BaseChatMesh {
+public:
+@STORAGE@
+  unsigned relocations = 0;
+  BaseChatMesh() {
+#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
+    contacts = contacts_fallback;
+    sort_array = sort_array_fallback;
+    contact_capacity = CONTACT_PSRAM_FALLBACK_TOTAL_SLOTS;
+#endif
+  }
+  ~BaseChatMesh() {
+#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
+    if (contacts != contacts_fallback) {
+      heap_caps_free(contacts);
+      heap_caps_free(sort_array);
+    }
+#endif
+  }
+  static void resetContactValue(ContactInfo& contact) { contact = ContactInfo(); }
+  void onContactReferenceChanged(const ContactInfo*, ContactInfo*) { ++relocations; }
+  bool initializeContactStorage();
+  int searchPeersByHash(const uint8_t* hash);
+  void scanRecentContacts(int last_n, ContactVisitor* visitor);
+};
+@METHODS@
+struct Visitor : ContactVisitor {
+  std::vector<ContactInfo> seen;
+  void onContactVisit(const ContactInfo& contact) override { seen.push_back(contact); }
+};
+static void checkSort(BaseChatMesh& mesh, int requested, size_t expected) {
+  Visitor visitor;
+  mesh.scanRecentContacts(requested, &visitor);
+  assert(visitor.seen.size() == expected);
+  std::set<uint32_t> ordinals;
+  for (size_t i = 0; i < visitor.seen.size(); ++i) {
+    const auto& contact = visitor.seen[i];
+    assert(contact.id.ordinal < static_cast<uint32_t>(mesh.num_contacts));
+    assert(ordinals.insert(contact.id.ordinal).second);
+    if (i > 0) {
+      const auto& previous = visitor.seen[i - 1];
+      const bool previous_favorite = (previous.flags & 1) != 0;
+      const bool favorite = (contact.flags & 1) != 0;
+      assert(previous_favorite || !favorite);
+      if (previous_favorite == favorite)
+        assert(previous.last_advert_timestamp >= contact.last_advert_timestamp);
+    }
+  }
+}
+static void checkFullTable() {
+  auto mesh = std::make_unique<BaseChatMesh>();
+  assert(mesh->initializeContactStorage());
+  mesh->num_contacts = MAX_CONTACTS + MAX_ANON_CONTACTS;
+  for (int i = 0; i < mesh->num_contacts; ++i) {
+    mesh->contacts[i].id.ordinal = i;
+    mesh->contacts[i].flags = (i % 7 == 0 ? 1 : 0) | 0x20;
+    mesh->contacts[i].last_advert_timestamp = (static_cast<uint32_t>(i) * 7919) % 997;
+  }
+  checkSort(*mesh, 0, mesh->num_contacts);
+  checkSort(*mesh, 11, 11);
+  checkSort(*mesh, mesh->num_contacts + 1, mesh->num_contacts);
+
+  // Twenty late persistent matches precede the eight transient matches. The
+  // 24-result limit must preserve the highest index and then four transients.
+  const uint8_t hash = 0x72;
+  for (int i = 0; i < MAX_ANON_CONTACTS; ++i) mesh->contacts[i].id.prefix = hash;
+  for (int i = mesh->num_contacts - 20; i < mesh->num_contacts; ++i)
+    mesh->contacts[i].id.prefix = hash;
+  assert(mesh->searchPeersByHash(&hash) == MAX_SEARCH_RESULTS);
+  for (int i = 0; i < 20; ++i)
+    assert(mesh->matching_peer_indexes[i] == mesh->num_contacts - 20 + i);
+  for (int i = 20; i < MAX_SEARCH_RESULTS; ++i)
+    assert(mesh->matching_peer_indexes[i] == i - 20);
+  for (int i = mesh->num_contacts - 24; i < mesh->num_contacts - 20; ++i)
+    mesh->contacts[i].id.prefix = hash;
+  assert(mesh->searchPeersByHash(&hash) == MAX_SEARCH_RESULTS);
+  for (int i = 0; i < MAX_SEARCH_RESULTS; ++i)
+    assert(mesh->matching_peer_indexes[i] == mesh->num_contacts - 24 + i);
+  const uint8_t missing = 0x91;
+  assert(mesh->searchPeersByHash(&missing) == 0);
+}
+#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
+static void checkPsram() {
+  for (unsigned fail : {1U, 2U, 0U}) {
+    allocations = 0;
+    fail_allocation = fail;
+    allocation_sizes.clear();
+    {
+      BaseChatMesh mesh;
+      mesh.num_contacts = MAX_ANON_CONTACTS + 16;
+      for (int i = 0; i < mesh.num_contacts; ++i) mesh.contacts[i].id.ordinal = i + 100;
+      assert(mesh.initializeContactStorage() == (fail == 0));
+      assert(allocations == 2);
+      assert(allocation_sizes[1].first == MAX_CONTACTS + MAX_ANON_CONTACTS);
+      assert(allocation_sizes[1].second == sizeof(ContactIndex));
+      assert(mesh.contact_capacity == (fail ? MAX_ANON_CONTACTS + 16
+                                            : MAX_CONTACTS + MAX_ANON_CONTACTS));
+      assert((mesh.contacts == mesh.contacts_fallback) == (fail != 0));
+      assert((mesh.sort_array == mesh.sort_array_fallback) == (fail != 0));
+      for (int i = 0; i < mesh.num_contacts; ++i) assert(mesh.contacts[i].id.ordinal == i + 100);
+      if (!fail) {
+        const unsigned before = allocations;
+        assert(mesh.initializeContactStorage());
+        assert(allocations == before);
+      }
+    }
+    assert(outstanding.empty());
+  }
+  fail_allocation = 0;
+}
+#endif
+int main() {
+#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535
+  static_assert(std::is_same<ContactIndex, uint16_t>::value, "compact contact indices");
+#else
+  static_assert(std::is_same<ContactIndex, int>::value, "retain wide contact indices");
+#endif
+  static_assert(sizeof(((BaseChatMesh*)nullptr)->matching_peer_indexes)
+                == MAX_SEARCH_RESULTS * sizeof(ContactIndex), "matching index storage");
+#if defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)
+  static_assert(std::is_same<decltype(BaseChatMesh::sort_array), ContactIndex*>::value,
+                "PSRAM sort allocation type");
+  static_assert(sizeof(((BaseChatMesh*)nullptr)->sort_array_fallback)
+                == (MAX_ANON_CONTACTS + 16) * sizeof(ContactIndex), "fallback index storage");
+  checkPsram();
+#else
+  static_assert(sizeof(((BaseChatMesh*)nullptr)->sort_array)
+                == (MAX_CONTACTS + MAX_ANON_CONTACTS) * sizeof(ContactIndex), "sort index storage");
+#endif
+  checkFullTable();
+  assert(outstanding.empty());
+  puts("PASS: contact sorting, peer collisions, widths and allocation ownership");
+}
+'''
+
+
+class ContactIndexTests(unittest.TestCase):
+    def test_contact_layout_on_all_platforms(self):
+        # Compile the real record rather than the indexing harness's small
+        # stand-in. Only cached nRF52 records have the extra storage-slot field
+        # whose placement can cost four bytes per contact.
+        compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++")
+        self.assertIsNotNone(compiler, "a host C++17 compiler is required")
+        for platform in ("NRF52_PLATFORM", "ESP32_PLATFORM", "RP2040_PLATFORM", "STM32_PLATFORM"):
+            for cache in (0, 1):
+                expected = 92 if cache else (188 if platform == "NRF52_PLATFORM" else 184)
+                with self.subTest(platform=platform, cache=cache):
+                    source = ("#include <helpers/ContactInfo.h>\n"
+                              f'static_assert(sizeof(ContactInfo) == {expected}, "contact RAM budget");\n')
+                    compiled = subprocess.run([
+                        compiler, "-std=c++17", "-fsyntax-only", "-x", "c++", "-",
+                        f"-D{platform}=1", f"-DMESH_CONTACT_CACHE={cache}",
+                        f"-I{ROOT / 'test/fixtures/contact_cache/mocks'}",
+                        f"-I{ROOT / 'lib/ed25519'}", f"-I{ROOT / 'src'}",
+                    ], input=source, capture_output=True, text=True, timeout=30)
+                    self.assertEqual(compiled.returncode, 0, compiled.stdout + compiled.stderr)
+
+    def test_production_indices(self):
+        compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++")
+        self.assertIsNotNone(compiler, "a host C++17 compiler is required")
+        header = (ROOT / "src/helpers/BaseChatMesh.h").read_text()
+        source = (ROOT / "src/helpers/BaseChatMesh.cpp").read_text()
+        index_types = "\n".join(re.findall(
+            r"^#define MAX_(?:ANON_CONTACTS|SEARCH_RESULTS)\s+.+$", header, re.MULTILINE))
+        index_types += "\n" + header[header.index("#if MAX_CONTACTS + MAX_ANON_CONTACTS <= 65535"):
+                                     header.index("#ifndef MAX_CONNECTIONS")]
+        storage_start = header.index("#if defined(ESP32_PLATFORM)",
+                                     header.index("friend class ContactsIterator;"))
+        storage = header[storage_start:header.index("  unsigned long txt_send_timeout;", storage_start)]
+        methods = "\n".join(extract_braced(source, signature) for signature in (
+            "bool BaseChatMesh::initializeContactStorage(",
+            "int BaseChatMesh::searchPeersByHash(",
+            "static int cmp_contact_list_order(",
+            "void BaseChatMesh::scanRecentContacts("))
+        methods = "static ContactInfo* table;\n" + methods
+        harness = (HARNESS.replace("@INDEX_TYPES@", index_types)
+                   .replace("@VISITOR@", extract_braced(header, "class ContactVisitor") + ";")
+                   .replace("@STORAGE@", storage).replace("@METHODS@", methods))
+        variants = ((350, None, False, False),
+                    (350, "NRF52_PLATFORM", False, True),
+                    (350, "ESP32_PLATFORM", False, True),
+                    (350, "ESP32_PLATFORM", True, False),
+                    (350, "ESP32_PLATFORM", True, True),
+                    (350, "RP2040_PLATFORM", False, False),
+                    (32, "STM32_PLATFORM", False, False),
+                    (65527, None, False, False), (65528, None, False, False),
+                    (65529, None, False, False), (65529, "ESP32_PLATFORM", True, True))
+        with tempfile.TemporaryDirectory(prefix="meshcore-contact-indices-") as directory:
+            work = Path(directory)
+            test_source = work / "test.cpp"
+            test_source.write_text(harness, encoding="ascii")
+            for contacts, platform, psram, cache in variants:
+                with self.subTest(contacts=contacts, platform=platform, psram=psram, cache=cache):
+                    binary = work / f"indices-{contacts}-{platform}-{psram}-{cache}"
+                    command = [compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror",
+                               "-Wno-unused-function", "-Wno-sign-compare",
+                               f"-DMAX_CONTACTS={contacts}", f"-DMESH_CONTACT_CACHE={int(cache)}",
+                               f"-I{ROOT / 'src'}", str(test_source), "-o", str(binary)]
+                    if platform:
+                        command.append(f"-D{platform}=1")
+                    if psram:
+                        command.append("-DBOARD_HAS_PSRAM=1")
+                    if sys.platform.startswith("linux"):
+                        command += ["-fsanitize=address,undefined", "-fno-sanitize-recover=all",
+                                    "-fno-pie", "-no-pie"]
+                    built = subprocess.run(command, capture_output=True, text=True, timeout=60)
+                    self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
+                    checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=20)
+                    self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
+                    self.assertIn("PASS: contact sorting", checked.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_morse_cli.py

@@ -54,13 +54,13 @@ class MorseCLITests(unittest.TestCase):
                     "const char* path, bool& present) { if (!fs || fs->fail_presence) return false; "
                     "present = fs->exists(path); return true; } }\n")
         transaction = (ROOT / "src/helpers/ContactFileTransaction.h").read_text(encoding="ascii").replace(
-            '#include "IdentityStore.h"', '#include <helpers/IdentityStore.h>')
+            '#include "IdentityStore.h"', '#include <helpers/IdentityStore.h>').replace(
+            '#include "PersistentStoreFormat.h"', '#include <helpers/PersistentStoreFormat.h>')
         headers = {
             "helpers/IdentityStore.h": fs_mock,
             "FilePresence.h": presence,
             "AtomicFileWriter.h": (ROOT / "src/helpers/AtomicFileWriter.h").read_text(encoding="ascii"),
             "ContactFileTransaction.h": transaction,
-            "PersistentStoreFormat.h": (ROOT / "src/helpers/PersistentStoreFormat.h").read_text(encoding="ascii"),
         }
         prefix = b"MC\x01\x19" + (902100).to_bytes(4, "little")
         golden = prefix + (zlib.crc32(prefix) ^ 0xffffffff).to_bytes(4, "little")

tools/bible/README.md

@@ -130,18 +130,28 @@ terminators, versus 98,911 bytes for the pinned UTF-8 source):
 
 | Block limit | Blocks | DEFLATE bytes | Index bytes | Total flash data | Savings |
 | --- | ---: | ---: | ---: | ---: | ---: |
-| 1 KiB | 101 | 47,133 | 1,212 | 48,345 | 49.94% |
-| 2 KiB | 49 | 42,187 | 588 | 42,775 | 55.70% |
-| **4 KiB** | **24** | **38,366** | **288** | **38,654** | **59.97%** |
+| 1 KiB | 101 | 46,945 | 808 | 47,753 | 50.55% |
+| 2 KiB | 49 | 42,074 | 392 | 42,466 | 56.02% |
+| **4 KiB** | **24** | **38,293** | **192** | **38,485** | **60.15%** |
 
 Savings in the table are relative to uncompressed ASCII. Google Zopfli 0.4.3
-uses 15 optimization iterations to write standards-compliant raw DEFLATE.
-The chosen 4 KiB limit saves **4,121 flash bytes** over the previous 2 KiB
-packing, including indexes, and uses another 2 KiB of temporary decode stack.
+uses 1,000 host optimization iterations to write standards-compliant raw DEFLATE.
+The chosen 4 KiB limit saves **3,981 flash bytes** versus 2 KiB packing at the
+same compression effort, including indexes, and uses another 2 KiB of temporary
+decode stack. The decode block size and stack do not increase in this revision.
 All 879 references, wording, canonical source digest, navigation and terminal
 behavior are retained. No 6-bit packing is used. These figures include
 block-index padding but exclude command code, attribution and the decoder.
 
+The generated index uses aligned 16-bit block offsets while every block start
+fits that range, and selects the original 32-bit representation automatically
+for larger corpora. The lookup is specialized at compile time, with no runtime
+format branch or packed/unaligned fields. The current 24 indexes use eight
+bytes each instead of twelve, saving 96 bytes. Higher compression saves another
+73 bytes versus the previous 15-iteration 4 KiB data, for **169 fewer data bytes**
+with the same plaintext, decoder and scratch capacity. Tests decode real blocks
+at offsets 65,535 and 65,536 to cover both representations and their bounds.
+
 Compression runs on the computer. Firmware reuses the existing bounds-checked
 C `tinf` decoder and expands one independent raw-DEFLATE block per lookup.
 The block index stores only each block's first verse; scanning NUL separators
@@ -153,9 +163,9 @@ the screen reader's small cursor object is described above. The temporary
 
 The native Reader suite also cross-compiles the production Reader and C
 `tinf` decoder with Cortex-M4 GCC 14.2 and `-fstack-usage`. With 4 KiB blocks,
-the terminal decode frames use at most 6,304 bytes including the bounded
+the terminal decode frames use at most 6,264 bytes including the bounded
 page wrapper; the adaptive-font/button Reader screen's local decode path is
-6,432 bytes. The check reserves another 1,536 bytes for outer callers within
+6,400 bytes. The check reserves another 1,536 bytes for outer callers within
 the unchanged 8 KiB nRF52 loop stack. This is a compiler-derived bound on
 these paths, not a physical stack high-water measurement. Decoder frames are
 counted in addition to output scratch; ordinary commands never reserve it.

tools/bible/pack_reader.py

@@ -9,11 +9,14 @@ import unicodedata
 
 ROOT = Path(__file__).resolve().parents[2]
 sys.path.insert(0, str(ROOT / "scripts"))
-from zopfli_compress import ZOPFLI_VERSION, raw_deflate_compress  # noqa: E402
+from zopfli_compress import (  # noqa: E402
+    MAXIMUM_ITERATIONS, ZOPFLI_VERSION, raw_deflate_compress,
+)
 
 SOURCE = Path(__file__).with_name("reader-web.json")
 HEADER = ROOT / "src/helpers/bible/ReaderData.generated.h"
 BLOCK_SIZE = 4096
+COMPRESSION_ITERATIONS = MAXIMUM_ITERATIONS
 ASCII_TYPOGRAPHY = str.maketrans({
     "\u00a0": " ", "\u2013": "-", "\u2014": "-",
     "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"',
@@ -74,31 +77,41 @@ def pack(document, block_size=BLOCK_SIZE):
         raise ValueError("source contains no verse text")
     packed, descriptors = bytearray(), []
     for first_verse, block in zip(starts, blocks):
-        compressed = raw_deflate_compress(block)
+        compressed = raw_deflate_compress(block, numiterations=COMPRESSION_ITERATIONS)
         descriptors.append((len(packed), len(compressed), len(block), first_verse))
         packed.extend(compressed)
     return bytes(packed), descriptors
 
 
+def block_index_layout(blocks):
+    """Use aligned halfword offsets when possible, retaining larger corpora."""
+    if not blocks or any(not 0 <= block[0] <= 0xFFFFFFFF for block in blocks):
+        raise ValueError("Reader block offsets must fit uint32_t")
+    return ("uint16_t", 8) if max(block[0] for block in blocks) <= 0xFFFF else ("uint32_t", 12)
+
+
 def render(document):
     packed, blocks = pack(document)
+    offset_type, index_bytes = block_index_layout(blocks)
     digest = hashlib.sha256(json.dumps(document, ensure_ascii=False, sort_keys=True,
                                      separators=(",", ":")).encode()).hexdigest()
     lines = ["// Generated by tools/bible/pack_reader.py; do not edit.",
-             f"// Compressed with Google Zopfli {ZOPFLI_VERSION} (15 iterations).",
+             f"// Compressed with Google Zopfli {ZOPFLI_VERSION} ({COMPRESSION_ITERATIONS} iterations).",
              "// Offline Reader corpus. Public domain; see tools/bible/README.md.",
              "// Firmware typography: ASCII quotes, apostrophes, dashes and spaces; wording unchanged.",
              f"// Canonical source SHA-256: {digest}",
              "#pragma once", '#include "ReaderLookup.h"',
              "namespace mesh { namespace bible { namespace generated {",
              f'static_assert(kBlockSize == {BLOCK_SIZE}, "Regenerate Reader data after changing block size");',
-             'static_assert(sizeof(BlockIndex) == 12, "Reader flash-size accounting expects a 12-byte index");',
+             f"using ReaderBlockIndex = BlockIndexT<{offset_type}>;",
+             f"using ReaderCorpus = CorpusT<{offset_type}>;",
+             f'static_assert(sizeof(ReaderBlockIndex) == {index_bytes}, "Reader flash-size accounting expects a {index_bytes}-byte index");',
              "static const uint8_t readerData[] = {"]
     for start in range(0, len(packed), 20):
         lines.append("  " + ",".join(f"0x{v:02x}" for v in packed[start:start + 20]) + ",")
-    lines += ["};", "static const BlockIndex readerBlocks[] = {"]
+    lines += ["};", "static const ReaderBlockIndex readerBlocks[] = {"]
     lines += ["  {%d, %d, %d, %d}," % block for block in blocks]
-    lines += ["};", "static const Corpus readerCorpus = {",
+    lines += ["};", "static const ReaderCorpus readerCorpus = {",
               f"  {json.dumps(document['translation'], ensure_ascii=False)},",
               f"  {json.dumps(document['attribution'], ensure_ascii=False)},",
               "  readerData, sizeof(readerData), readerBlocks,",
@@ -117,7 +130,7 @@ def main():
     if args.compare:
         for size in (1024, 2048, 4096):
             packed, blocks = pack(document, size)
-            tables = len(blocks) * 12
+            tables = len(blocks) * block_index_layout(blocks)[1]
             plain = sum(block[2] for block in blocks)
             print(f"{size} byte blocks: {len(blocks)} blocks, {plain} plain, "
                   f"{len(packed)} compressed, {tables} index, "
@@ -133,7 +146,7 @@ def main():
         args.output.write_text(header, encoding="utf-8", newline="\n")
     packed, blocks = pack(document)
     plain = sum(block[2] for block in blocks)
-    tables = len(blocks) * 12
+    tables = len(blocks) * block_index_layout(blocks)[1]
     print(f"Reader: {len(references())} verses, {len(blocks)} blocks; "
           f"{plain} plain bytes -> {len(packed)} DEFLATE + {tables} index bytes "
           f"({100 * (1 - (len(packed) + tables) / plain):.1f}% smaller incl. indexes)")