#!/usr/bin/env python3 """Pack the pinned public-domain source for the offline Reader into DEFLATE blocks.""" import argparse import hashlib import json from pathlib import Path import sys import unicodedata ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "scripts")) from zopfli_compress import ZOPFLI_VERSION, raw_deflate_compress # noqa: E402 SOURCE = Path(__file__).with_name("reader-web.json") HEADER = ROOT / "src/helpers/bible/ReaderData.generated.h" BLOCK_SIZE = 2048 ASCII_TYPOGRAPHY = str.maketrans({ "\u00a0": " ", "\u2013": "-", "\u2014": "-", "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"', }) CHAPTER_VERSES = (51, 25, 36, 54, 47, 71, 53, 59, 41, 42, 57, 50, 38, 31, 27, 33, 26, 40, 42, 31, 25) def references(): return [f"{c}:{v}" for c, count in enumerate(CHAPTER_VERSES, 1) for v in range(1, count + 1)] def checked_text(value, limit): if not isinstance(value, str) or not value or value != value.strip(): raise ValueError("expected nonempty trimmed text") if any(unicodedata.category(c).startswith("C") for c in value): raise ValueError("control/format characters are not allowed in terminal text") encoded = value.encode("utf-8") if len(encoded) > limit: raise ValueError(f"text exceeds {limit} UTF-8 bytes") return encoded def ascii_verse(value, limit): # Keep the pinned source exact. Only the generated firmware copy changes # typography; never transliterate letters or silently discard characters. checked_text(value, limit * 3) try: encoded = value.translate(ASCII_TYPOGRAPHY).encode("ascii") except UnicodeEncodeError as error: raise ValueError("unsupported non-ASCII character in Reader source") from error if len(encoded) > limit: raise ValueError(f"text exceeds {limit} ASCII bytes") return encoded def pack(document, block_size=BLOCK_SIZE): if block_size not in (1024, 2048, 4096): raise ValueError("block size must be 1024, 2048, or 4096") checked_text(document["translation"], 24) checked_text(document["attribution"], 256) verses = document["verses"] if not isinstance(verses, dict) or set(verses) != set(references()): raise ValueError("source must explicitly include all 879 Reader references") blocks, starts, current = [], [0], bytearray() for index, reference in enumerate(references()): verse = verses[reference] encoded = (ascii_verse(verse, block_size - 1) if verse is not None else b"") + b"\0" if len(current) + len(encoded) > block_size: blocks.append(bytes(current)) current.clear() starts.append(index) current.extend(encoded) if current: blocks.append(bytes(current)) if not blocks: raise ValueError("source contains no verse text") packed, descriptors = bytearray(), [] for first_verse, block in zip(starts, blocks): compressed = raw_deflate_compress(block) descriptors.append((len(packed), len(compressed), len(block), first_verse)) packed.extend(compressed) return bytes(packed), descriptors def render(document): packed, blocks = pack(document) digest = hashlib.sha256(json.dumps(document, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() lines = ["// Generated by tools/bible/pack_reader.py; do not edit.", f"// Compressed with Google Zopfli {ZOPFLI_VERSION} (15 iterations).", "// Offline Reader corpus. Public domain; see tools/bible/README.md.", "// Firmware typography: ASCII quotes, apostrophes, dashes and spaces; wording unchanged.", f"// Canonical source SHA-256: {digest}", "#pragma once", '#include "ReaderLookup.h"', "namespace mesh { namespace bible { namespace generated {", f'static_assert(kBlockSize == {BLOCK_SIZE}, "Regenerate Reader data after changing block size");', 'static_assert(sizeof(BlockIndex) == 12, "Reader flash-size accounting expects a 12-byte index");', "static const uint8_t readerData[] = {"] for start in range(0, len(packed), 20): lines.append(" " + ",".join(f"0x{v:02x}" for v in packed[start:start + 20]) + ",") lines += ["};", "static const BlockIndex readerBlocks[] = {"] lines += [" {%d, %d, %d, %d}," % block for block in blocks] lines += ["};", "static const Corpus readerCorpus = {", f" {json.dumps(document['translation'], ensure_ascii=False)},", f" {json.dumps(document['attribution'], ensure_ascii=False)},", " readerData, sizeof(readerData), readerBlocks,", " sizeof(readerBlocks) / sizeof(readerBlocks[0]), kVerseCount", "};", "}}}", ""] return "\n".join(lines) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source", type=Path, default=SOURCE) parser.add_argument("--output", type=Path, default=HEADER) parser.add_argument("--check", action="store_true") parser.add_argument("--compare", action="store_true", help="compare 1/2/4 KiB blocks without writing") args = parser.parse_args() document = json.loads(args.source.read_text(encoding="utf-8")) if args.compare: for size in (1024, 2048, 4096): packed, blocks = pack(document, size) tables = len(blocks) * 12 plain = sum(block[2] for block in blocks) print(f"{size} byte blocks: {len(blocks)} blocks, {plain} plain, " f"{len(packed)} compressed, {tables} index, " f"{len(packed) + tables} total, " f"{100 * (1 - (len(packed) + tables) / plain):.2f}% saved") return header = render(document) if args.check: if args.output.read_text(encoding="utf-8") != header: raise SystemExit("Reader header differs; regenerate with tools/bible/pack_reader.py") else: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(header, encoding="utf-8", newline="\n") packed, blocks = pack(document) plain = sum(block[2] for block in blocks) tables = len(blocks) * 12 print(f"Reader: {len(references())} verses, {len(blocks)} blocks; " f"{plain} plain bytes -> {len(packed)} DEFLATE + {tables} index bytes " f"({100 * (1 - (len(packed) + tables) / plain):.1f}% smaller incl. indexes)") if __name__ == "__main__": main()