HaloKeymind/tools/bible/pack_john.py

#!/usr/bin/env python3
"""Pack the pinned public-domain WEB John source into independent DEFLATE blocks."""
import argparse
import hashlib
import json
from pathlib import Path
import unicodedata
import zlib

ROOT = Path(__file__).resolve().parents[2]
SOURCE = Path(__file__).with_name("john-web.json")
HEADER = ROOT / "src/helpers/bible/JohnData.generated.h"
BLOCK_SIZE = 2048
ASCII_TYPOGRAPHY = str.maketrans({
    "\u00a0": " ", "\u2013": "-", "\u2014": "-",
    "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"',
})
CHAPTER_VERSES = (51, 25, 36, 54, 47, 71, 53, 59, 41, 42, 57,
                  50, 38, 31, 27, 33, 26, 40, 42, 31, 25)


def references():
    return [f"{c}:{v}" for c, count in enumerate(CHAPTER_VERSES, 1)
            for v in range(1, count + 1)]


def checked_text(value, limit):
    if not isinstance(value, str) or not value or value != value.strip():
        raise ValueError("expected nonempty trimmed text")
    if any(unicodedata.category(c).startswith("C") for c in value):
        raise ValueError("control/format characters are not allowed in terminal text")
    encoded = value.encode("utf-8")
    if len(encoded) > limit:
        raise ValueError(f"text exceeds {limit} UTF-8 bytes")
    return encoded


def ascii_verse(value, limit):
    # Keep the pinned source exact. Only the generated firmware copy changes
    # typography; never transliterate letters or silently discard characters.
    checked_text(value, limit * 3)
    try:
        encoded = value.translate(ASCII_TYPOGRAPHY).encode("ascii")
    except UnicodeEncodeError as error:
        raise ValueError("unsupported non-ASCII character in John source") from error
    if len(encoded) > limit:
        raise ValueError(f"text exceeds {limit} ASCII bytes")
    return encoded


def pack(document, block_size=BLOCK_SIZE):
    if block_size not in (1024, 2048, 4096):
        raise ValueError("block size must be 1024, 2048, or 4096")
    checked_text(document["translation"], 24)
    checked_text(document["attribution"], 256)
    verses = document["verses"]
    if not isinstance(verses, dict) or set(verses) != set(references()):
        raise ValueError("source must explicitly include all 879 John references")
    blocks, starts, current = [], [0], bytearray()
    for index, reference in enumerate(references()):
        verse = verses[reference]
        encoded = (ascii_verse(verse, block_size - 1) if verse is not None else b"") + b"\0"
        if len(current) + len(encoded) > block_size:
            blocks.append(bytes(current))
            current.clear()
            starts.append(index)
        current.extend(encoded)
    if current:
        blocks.append(bytes(current))
    if not blocks:
        raise ValueError("source contains no verse text")
    packed, descriptors = bytearray(), []
    for first_verse, block in zip(starts, blocks):
        compressor = zlib.compressobj(level=9, wbits=-15)
        compressed = compressor.compress(block) + compressor.flush()
        descriptors.append((len(packed), len(compressed), len(block), first_verse))
        packed.extend(compressed)
    return bytes(packed), descriptors


def render(document):
    packed, blocks = pack(document)
    digest = hashlib.sha256(json.dumps(document, ensure_ascii=False, sort_keys=True,
                                     separators=(",", ":")).encode()).hexdigest()
    lines = ["// Generated by tools/bible/pack_john.py; do not edit.",
             "// World English Bible (engwebp), John. Public domain; see tools/bible/README.md.",
             "// Firmware typography: ASCII quotes, apostrophes, dashes and spaces; wording unchanged.",
             f"// Canonical source SHA-256: {digest}",
             "#pragma once", '#include "JohnLookup.h"',
             "namespace mesh { namespace bible { namespace generated {",
             f'static_assert(kBlockSize == {BLOCK_SIZE}, "Regenerate John data after changing block size");',
             'static_assert(sizeof(BlockIndex) == 12, "John flash-size accounting expects a 12-byte index");',
             "static const uint8_t johnData[] = {"]
    for start in range(0, len(packed), 20):
        lines.append("  " + ",".join(f"0x{v:02x}" for v in packed[start:start + 20]) + ",")
    lines += ["};", "static const BlockIndex johnBlocks[] = {"]
    lines += ["  {%d, %d, %d, %d}," % block for block in blocks]
    lines += ["};", "static const Corpus johnCorpus = {",
              f"  {json.dumps(document['translation'], ensure_ascii=False)},",
              f"  {json.dumps(document['attribution'], ensure_ascii=False)},",
              "  johnData, sizeof(johnData), johnBlocks,",
              "  sizeof(johnBlocks) / sizeof(johnBlocks[0]), kVerseCount", "};", "}}}", ""]
    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source", type=Path, default=SOURCE)
    parser.add_argument("--output", type=Path, default=HEADER)
    parser.add_argument("--check", action="store_true")
    parser.add_argument("--compare", action="store_true", help="compare 1/2/4 KiB blocks without writing")
    args = parser.parse_args()
    document = json.loads(args.source.read_text(encoding="utf-8"))
    if args.compare:
        for size in (1024, 2048, 4096):
            packed, blocks = pack(document, size)
            tables = len(blocks) * 12
            plain = sum(block[2] for block in blocks)
            print(f"{size} byte blocks: {len(blocks)} blocks, {plain} plain, "
                  f"{len(packed)} compressed, {tables} index, "
                  f"{len(packed) + tables} total, "
                  f"{100 * (1 - (len(packed) + tables) / plain):.2f}% saved")
        return
    header = render(document)
    if args.check:
        if args.output.read_text(encoding="utf-8") != header:
            raise SystemExit("John header differs; regenerate with tools/bible/pack_john.py")
    else:
        args.output.parent.mkdir(parents=True, exist_ok=True)
        args.output.write_text(header, encoding="utf-8", newline="\n")
    packed, blocks = pack(document)
    plain = sum(block[2] for block in blocks)
    tables = len(blocks) * 12
    print(f"John: {len(references())} verses, {len(blocks)} blocks; "
          f"{plain} plain bytes -> {len(packed)} DEFLATE + {tables} index bytes "
          f"({100 * (1 - (len(packed) + tables) / plain):.1f}% smaller incl. indexes)")


if __name__ == "__main__":
    main()