Pack unchanged Reader text more tightly for Full firmware

examples/companion_radio/MyMesh.cpp

@@ -11520,7 +11520,7 @@ void MyMesh::loop() {
 #endif
   servicePendingSerialReply();
 #if COMPANION_FEATURE_READER
-  // Every verse chunk reinflates its 2 KiB flash block. Retain the cursor but
+  // Every verse chunk reinflates its 4 KiB flash block. Retain the cursor but
   // defer that optional decode/output work until envelope timing is finished.
   if (!mesh::morseTimingActive(_radio)) mesh::serviceReaderCommand();
 #endif

examples/companion_radio/ui-new/ReaderScreen.h

@@ -7,7 +7,7 @@
 #endif
 
 // Included by UITask.cpp after the task and MyMesh declarations. Only a small
-// cursor/checkpoint object persists; the 2 KiB decode buffer is on demand.
+// cursor/checkpoint object persists; the 4 KiB decode buffer is on demand.
 class ReaderScreen : public UIScreen {
   UITask* _task;
   DisplayDriver* _display;

src/helpers/CompanionReader.cpp

@@ -143,7 +143,7 @@ void formatVerseReply(Selection selection, uint8_t page, const char* text,
   memcpy(reply + prefix_len + text_len, next_command, suffix_len + 1);
 }
 
-// Only a recognized Reader request enters this 2 KiB decode frame. Normal
+// Only a recognized Reader request enters this 4 KiB decode frame. Normal
 // framed commands must not reserve it, including with link-time optimization.
 __attribute__((noinline)) void replyVerse(Selection selection, uint8_t page,
                                         char* reply, size_t capacity) {

src/helpers/bible/ReaderLookup.h

@@ -6,7 +6,7 @@
 
 namespace mesh { namespace bible {
 
-static constexpr size_t kBlockSize = 2048;
+static constexpr size_t kBlockSize = 4096;
 static constexpr uint16_t kVerseCount = 879;
 static constexpr uint8_t kChapterVerses[] = {
   51, 25, 36, 54, 47, 71, 53, 59, 41, 42, 57,

test/fixtures/companion_reader/test_reader_screen.cpp

@@ -442,7 +442,7 @@ int main(int argc, char** argv) {
       record[i] ^= 1; assert(!decodeBookmark(record, decoded)); record[i] ^= 1;
     }
   }
-  for (Position p : {pos(0), pos(879), pos(1, 2048)}) {
+  for (Position p : {pos(0), pos(879), pos(1, kBlockSize)}) {
     encodeBookmark(p, record); assert(!decodeBookmark(record, saved));
   }
 

test/test_companion_reader.py

@@ -46,6 +46,69 @@ class CompanionReaderTest(unittest.TestCase):
                 actual.extend(part.decode("ascii") for part in plain[:-1].split(b"\0"))
             self.assertEqual(actual, list(self.expected_verses.values()))
 
+    def test_arm_reader_decode_frames_fit_existing_loop_stack(self):
+        # Use the actual Cortex-M4 compiler when installed. This does not
+        # invoke PlatformIO or reserve decode buffers in ordinary commands.
+        tool = shutil.which("arm-none-eabi-g++")
+        if not tool:
+            candidate = Path.home() / ".platformio/packages/toolchain-gccarmnoneeabi/bin/arm-none-eabi-g++"
+            if candidate.is_file():
+                tool = str(candidate)
+        if not tool:
+            self.skipTest("ARM compiler unavailable; firmware stack qualification remains required")
+        cc = str(Path(tool).with_name("arm-none-eabi-gcc"))
+        with tempfile.TemporaryDirectory(prefix="mesh-reader-arm-stack-") as directory:
+            directory = Path(directory)
+            (directory / "Arduino.h").write_text(
+                "#pragma once\n#include <stdint.h>\n#include <stddef.h>\n#include <stdio.h>\n#include <string.h>\n"
+                "class Stream { public: virtual size_t write(const uint8_t*,size_t);size_t print(const char*); };\n")
+            flags = ["-mcpu=cortex-m4", "-mthumb", "-Oz", "-fno-inline-small-functions",
+                     "-fstack-usage", "-DNRF52_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1",
+                     "-DENABLE_USB_INTERFACE=1", "-I" + str(directory),
+                     "-I" + str(ROOT / "src"), "-I" + str(FIXTURE)]
+            self.run_checked([tool, "-std=gnu++17", "-fno-exceptions", "-fno-rtti", *flags,
+                "-c", str(ROOT / "src/helpers/CompanionReader.cpp"), "-o", str(directory / "reader.o")])
+            self.run_checked([cc, "-std=gnu11", *flags, "-c",
+                str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(directory / "tinf.o")])
+
+            def frames(name):
+                result = {}
+                for line in (directory / (name + ".su")).read_text().splitlines():
+                    function, count, kind = line.rsplit("\t", 2)
+                    self.assertEqual(kind, "static", line)
+                    result[function] = int(count)
+                return result
+
+            def frame(rows, signature):
+                found = [count for function, count in rows.items() if signature in function and " [with " not in function]
+                self.assertEqual(len(found), 1, signature)
+                return found[0]
+
+            reader, decoder = frames("reader"), frames("tinf")
+            # Summing all helper frames overbounds mutually exclusive branches
+            # (tree building and block decoding), including their leaf calls.
+            decoder_bytes = sum(count for name, count in decoder.items()
+                if not name.endswith(":tinf_init") and not name.endswith(":tinf_uncompress"))
+            lookup = frame(reader, "mesh::readReaderVerse(")
+            terminal = (frame(reader, "mesh::handleReaderCommand(")
+                + frame(reader, "::printReplyPage(") + frame(reader, "::replyVerse(")
+                + lookup + decoder_bytes)
+            loop_bytes, caller_reserve = 8192, 1536
+            self.assertLessEqual(terminal + caller_reserve, loop_bytes)
+            ui_max = 0
+            for small_font in (0, 1):
+                name = "screen" + str(small_font)
+                self.run_checked([tool, "-std=gnu++17", "-fno-exceptions", "-fno-rtti", *flags,
+                    f"-DUI_SMALL_MESSAGE_FONT={small_font}", "-DUI_BUTTON_READER_HINT=1",
+                    "-DUI_READER_TOUCH_BAR=0", "-c", str(FIXTURE / "test_reader_screen.cpp"),
+                    "-o", str(directory / (name + ".o"))])
+                ui = frame(frames(name), "ReaderScreen::process(") + lookup + decoder_bytes
+                ui_max = max(ui_max, ui)
+                self.assertLessEqual(ui + caller_reserve, loop_bytes)
+            self.assertEqual(packer.BLOCK_SIZE, 4096)
+            print(f"ARM Reader local stack: terminal {terminal}, screen {ui_max}; "
+                  f"caller reserve {caller_reserve}, loop {loop_bytes} bytes")
+
     def test_ascii_typography_preserves_words_case_and_pinned_source(self):
         original = copy.deepcopy(self.document)
         sample = '\u201creader\u2019s\u00a0Word\u201d\u2014\u2018Test\u2019\u2013Yes!'
@@ -62,12 +125,12 @@ class CompanionReaderTest(unittest.TestCase):
     def test_ascii_data_size(self):
         packed, blocks = packer.pack(self.document)
         self.assertEqual(sum(block[2] for block in blocks), 96568)
-        self.assertEqual(len(blocks), 49)
-        self.assertEqual(len(packed) + 12 * len(blocks), 42775)
+        self.assertEqual(len(blocks), 24)
+        self.assertEqual(len(packed) + 12 * len(blocks), 38654)
 
     def test_invalid_sources_are_rejected(self):
         for bad_text in ("", "leading\ncommand", "bad\x1b[2J", "bad\0text",
-                         "bad\u202eRTL", " too much space", "a" * 2048,
+                         "bad\u202eRTL", " too much space", "a" * packer.BLOCK_SIZE,
                          "caf\u00e9", "unmapped\u2026punctuation", "bad\U0001f600text"):
             document = copy.deepcopy(self.document)
             document["verses"]["1:1"] = bad_text
@@ -191,6 +254,7 @@ int main() {
 struct Radio { bool active = false; };
 namespace mesh {
 bool morseTimingActive(const Radio* radio) { return radio && radio->active; }
+namespace power { bool handleRegulatorCommand(const char*,char*,size_t) { return false; } }
 }
 @PREDICATE@
 struct MyMesh {
@@ -220,7 +284,7 @@ int main() {
   assert(!mesh.handleCommand("get reader 3:16", 123, out.reply));
   assert(!mesh.handleCommand("AA|get reader 3:16", 123, out.reply));
   assert(!mesh.handleCommand("get name", 0, out.reply));
-  // Block synchronous 2 KiB decompression before it can stall Morse sampling,
+  // Block synchronous 4 KiB decompression before it can stall Morse sampling,
   // including tagged bounded replies. Ordinary readers resume unchanged.
   mesh.radio.active = true;
   assert(mesh.handleCommand("E5|get reader 21:17 2", 0, out.reply));

tools/bible/README.md

@@ -86,7 +86,7 @@ replacement and `.bak` fallback for interrupted saves. It does not change the
 radio-preference layout, contacts, channels, or keys. A malformed bookmark
 falls back to the previous valid record or the first reference; no filesystem is formatted.
 The reader allocates only a small cursor object on first use, reusing the
-same flash corpus and on-demand 2 KiB decode scratch as the terminal.
+same flash corpus and on-demand 4 KiB decode scratch as the terminal.
 
 ## Source and reproduction
 
@@ -131,21 +131,16 @@ terminators, versus 98,911 bytes for the pinned UTF-8 source):
 | Block limit | Blocks | DEFLATE bytes | Index bytes | Total flash data | Savings |
 | --- | ---: | ---: | ---: | ---: | ---: |
 | 1 KiB | 101 | 47,133 | 1,212 | 48,345 | 49.94% |
-| **2 KiB** | **49** | **42,187** | **588** | **42,775** | **55.70%** |
-| 4 KiB | 24 | 38,366 | 288 | 38,654 | 59.97% |
+| 2 KiB | 49 | 42,187 | 588 | 42,775 | 55.70% |
+| **4 KiB** | **24** | **38,366** | **288** | **38,654** | **59.97%** |
 
 Savings in the table are relative to uncompressed ASCII. Google Zopfli 0.4.3
-uses 15 optimization iterations to write standards-compliant raw DEFLATE, so
-the existing firmware decoder and RAM use do not change. At the chosen 2 KiB
-limit, the Zopfli asset is **2,097 bytes (4.67%)** smaller than the prior
-44,872-byte UTF-8 level-9 DEFLATE data and index; that result includes the
-existing ASCII typography conversion. No 6-bit packing is used.
-
-The chosen 2 KiB limit saves 5,570 flash bytes over 1 KiB for another 1 KiB
-of temporary decode storage. Going to 4 KiB saves only another 4,121 flash
-bytes and doubles that buffer. The 2 KiB choice leaves more stack headroom
-for nRF52's existing BLE/USB/radio calls. These figures include block-index
-padding but exclude the small command code, attribution strings and decoder.
+uses 15 optimization iterations to write standards-compliant raw DEFLATE.
+The chosen 4 KiB limit saves **4,121 flash bytes** over the previous 2 KiB
+packing, including indexes, and uses another 2 KiB of temporary decode stack.
+All 879 references, wording, canonical source digest, navigation and terminal
+behavior are retained. No 6-bit packing is used. These figures include
+block-index padding but exclude command code, attribution and the decoder.
 
 Compression runs on the computer. Firmware reuses the existing bounds-checked
 C `tinf` decoder and expands one independent raw-DEFLATE block per lookup.
@@ -154,22 +149,19 @@ inside the decoded block avoids a large per-verse index. Text/indexes remain
 in memory-mapped flash on ESP32 and nRF52. There is no permanent text buffer,
 cache, or additional compression library. The terminal needs no heap allocation;
 the screen reader's small cursor object is described above. The temporary
-2 KiB output buffer is **in addition to** the decoder's trees and call stack.
+4 KiB output buffer is **in addition to** the decoder's trees and call stack.
 
-A Cortex-M4 GCC 14.2 `-Os -fstack-usage` build of the RAK3401 Full Companion
-reports a 2,088-byte frame for verse output, a 16-byte command dispatcher,
-and about 1,720 bytes for the nested decoder calls. Including the terminal
-and main-loop frames, the decode path is approximately **4.3 KiB**, before
-framework/task and interrupt overhead; the nRF52 loop task has an 8 KiB stack.
-Before the navigation-hint addition, the small-font reader's decode/draw
-function had a 2,480-byte frame on RAK3401
-and a 2,496-byte frame on V4. Its nested decode path is approximately 4.3 KiB
-on RAK3401 and 4.7 KiB on V4, including the main-loop/UI frames but before
-framework/task and interrupt overhead. The ESP32-S3 V4 terminal's corresponding
-decode path is approximately 4.6 KiB.
-This is a compiler-derived estimate, not a hardware stack high-water reading.
-Ordinary non-reader commands never enter the decode-buffer function. Its object
-file has zero `.data`/`.bss`; the screen reader adds only its small UI state.
+The native Reader suite also cross-compiles the production Reader and C
+`tinf` decoder with Cortex-M4 GCC 14.2 and `-fstack-usage`. With 4 KiB blocks,
+the terminal decode frames use at most 6,304 bytes including the bounded
+page wrapper; the adaptive-font/button Reader screen's local decode path is
+6,432 bytes. The check reserves another 1,536 bytes for outer callers within
+the unchanged 8 KiB nRF52 loop stack. This is a compiler-derived bound on
+these paths, not a physical stack high-water measurement. Decoder frames are
+counted in addition to output scratch; ordinary commands never reserve it.
+The terminal retains only its existing cursor between calls, with no heap
+buffer or text cache. Linked firmware and physical stack checks remain
+separate from this native source-level bound.
 
 Tests compare all 879 decoded verses and terminal responses with an independently
 ASCII-normalized copy of the pinned plaintext source, check references across

tools/bible/pack_reader.py

@@ -13,7 +13,7 @@ from zopfli_compress import ZOPFLI_VERSION, raw_deflate_compress  # noqa: E402
 
 SOURCE = Path(__file__).with_name("reader-web.json")
 HEADER = ROOT / "src/helpers/bible/ReaderData.generated.h"
-BLOCK_SIZE = 2048
+BLOCK_SIZE = 4096
 ASCII_TYPOGRAPHY = str.maketrans({
     "\u00a0": " ", "\u2013": "-", "\u2014": "-",
     "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"',