Merge PR #13 with gzip-only WebConfig assets

Keep the graceful final-ACK TCP close and optional FEM control guards from
pixitha's browser compatibility fixes. Replace Brotli negotiation and duplicate
page assets with one gzip representation compressed by Google Zopfli 0.4.3
using 1,000 iterations. Compress the bootstrap, captive success, compact OTA,
and static WiFi setup pages with the same validated build cache.

Honor gzip quality values and wildcard precedence. Validate compressed OTA
page evidence in the firmware capability gate, and add native and Python
regressions plus CI coverage. Remove the unused Brotli build dependency.

Validation: 54 native tests, 56 Python/browser checks, full V4.3 build and
verified application flash. Actual V4 WiFi responses matched the generated
gzip assets; 30 repeat downloads completed without reboot. Chromium loaded
the setup and settings pages without script or transfer errors. Physical
Android testing is pending because the Pi detects no phone over USB or mDNS.

scripts/brotli_compress.py

@@ -1,31 +0,0 @@
-"""Pinned Google Brotli helper for the browser-decoded WebConfig asset."""
-
-from __future__ import annotations
-
-BROTLI_VERSION = "1.2.0"
-QUALITY = 11
-# Maximum RFC 7932/browser-compatible Brotli window (16 MiB). The page is
-# smaller than the default 4 MiB window, but pin the requested maximum anyway.
-LGWIN = 24
-
-try:
-    import brotli
-except ImportError as error:
-    raise RuntimeError(
-        "Google Brotli 1.2.0 is required to generate the MeshCore WebConfig "
-        "asset. Install it with 'python -m pip install -r "
-        "requirements-build.txt'."
-    ) from error
-
-if brotli.__version__ != BROTLI_VERSION:
-    raise RuntimeError(
-        "MeshCore requires Google Brotli %s, but %s is installed. Run "
-        "'python -m pip install -r requirements-build.txt'."
-        % (BROTLI_VERSION, brotli.__version__)
-    )
-
-
-def compress_html(data: bytes) -> bytes:
-    """Return a deterministic, maximum-quality RFC 7932 Brotli text stream."""
-    return brotli.compress(data, mode=brotli.MODE_TEXT, quality=QUALITY,
-                           lgwin=LGWIN)

scripts/check_firmware_capabilities.py

@@ -10,6 +10,7 @@ from pathlib import Path
 import sys
 import struct
 import zipfile
+import zlib
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools" / "mota"))
 from motalib import parse_nrf52_layout
@@ -74,6 +75,11 @@ def parse_args() -> argparse.Namespace:
         metavar="CAPABILITY=TEXT",
         help="Require TEXT to be present in the linked image.",
     )
+    parser.add_argument(
+        "--expect-gzip", action="append", default=[],
+        metavar="CAPABILITY=TEXT",
+        help="Require TEXT inside a complete gzip asset in the linked image.",
+    )
     parser.add_argument(
         "--expect-application", action="append", default=[],
         metavar="CAPABILITY=TEXT",
@@ -86,6 +92,24 @@ def stable_unique(values: list[str]) -> list[str]:
     return list(dict.fromkeys(value for value in values if value))
 
 
+def linked_gzip_assets(image):
+    """Inspect bounded, CRC-validated assets; ELF strings alone are not proof."""
+    assets = []
+    offset = 0
+    while True:
+        offset = image.find(b"\x1f\x8b\x08", offset)
+        if offset < 0:
+            return assets
+        decoder = zlib.decompressobj(31)
+        try:
+            decoded = decoder.decompress(image[offset:], 1024 * 1024 + 1)
+            if decoder.eof and len(decoded) <= 1024 * 1024:
+                assets.append(decoded)
+        except zlib.error:
+            pass
+        offset += 3
+
+
 def read_application(args):
     if args.firmware_bin is not None:
         return args.firmware_bin.read_bytes()
@@ -169,6 +193,8 @@ def main() -> int:
             malformed = True
     expectations = [(value, image, "linked image") for value in args.expect]
     expectations += [(value, application, "packaged application") for value in args.expect_application]
+    gzip_assets = linked_gzip_assets(image) if args.expect_gzip else []
+    expectations += [(value, gzip_assets, "linked gzip asset") for value in args.expect_gzip]
     for expectation, content, source in expectations:
         if "=" not in expectation:
             print(
@@ -178,7 +204,9 @@ def main() -> int:
             malformed = True
             continue
         capability, needle = expectation.split("=", 1)
-        present = bool(needle) and needle.encode("utf-8") in content
+        encoded = needle.encode("utf-8")
+        present = bool(needle) and (any(encoded in asset for asset in content)
+                                  if source == "linked gzip asset" else encoded in content)
         checks.append(
             {
                 "capability": capability,

scripts/generate_webconfig_html.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 #
-# Pre-build script: Brotli-compress webui/index.html into a PROGMEM C header so the
+# Pre-build script: compress webui/index.html into a PROGMEM C header so the
 # webconfig portal can serve the page straight from flash with
-# Content-Encoding: br. The generated header is tracked so ESP32 targets
+# Content-Encoding: gzip. The generated header is tracked so ESP32 targets
 # that replace extra_scripts still compile from a clean checkout; this script
 # rewrites it only when the emitted firmware bytes actually change.
 #
@@ -11,8 +11,8 @@
 # flash, so the page ships smaller than it reads.
 #
 # Output: src/helpers/esp32/WebConfigHtml.h
-#   WEBCONFIG_HTML_LOADER[] - small bootstrap page (PROGMEM)
-#   WEBCONFIG_HTML_BR[]     - complete Brotli-compressed page (PROGMEM)
+#   WEBCONFIG_HTML_LOADER[] - gzip-compressed bootstrap page (PROGMEM)
+#   WEBCONFIG_HTML_GZ[]     - complete gzip-compressed page (PROGMEM)
 #   WEBCONFIG_HTML_ETAG     - quoted strong ETag for the loader page
 #
 # Runnable outside SCons to inspect exactly what gets shipped:
@@ -20,6 +20,10 @@
 # or served directly by the mock backend with its --minify flag.
 
 import hashlib
+import gzip
+from pathlib import Path
+import tempfile
+import zlib
 import os
 import sys
 
@@ -35,7 +39,7 @@ HASH_MARKER = "// shipped-page-sha256: "
 
 sys.path.insert(0, os.path.join(os.getcwd(), "scripts"))
 from webconfig_minify import check_stripped, strip_source  # noqa: E402
-from brotli_compress import BROTLI_VERSION, LGWIN, QUALITY, compress_html  # noqa: E402
+from zopfli_compress import ZOPFLI_VERSION, MAXIMUM_ITERATIONS, gzip_compress  # noqa: E402
 
 
 def status(msg):
@@ -73,6 +77,27 @@ def append_byte_array(lines, name, data):
     lines.append("};")
 
 
+def compress_page(raw):
+    """Cache validated 1,000-iteration output so normal builds stay fast."""
+    key = hashlib.sha256((ZOPFLI_VERSION + ":" + str(MAXIMUM_ITERATIONS)).encode()
+                         + raw).hexdigest()
+    cache = Path(".pio/gzip-html")
+    path = cache / (key + ".gz")
+    try:
+        encoded = path.read_bytes()
+        if gzip.decompress(encoded) == raw:
+            return encoded
+    except (OSError, EOFError, gzip.BadGzipFile, zlib.error):
+        pass
+    encoded = gzip_compress(raw, numiterations=MAXIMUM_ITERATIONS)
+    cache.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=cache, delete=False) as output:
+        output.write(encoded)
+        temporary = output.name
+    os.replace(temporary, path)
+    return encoded
+
+
 def main():
     if not os.path.isfile(SOURCE):
         status("ERROR: %s not found" % SOURCE)
@@ -86,12 +111,27 @@ def main():
         status("%s -> %s (%d -> %d bytes)" % (SOURCE, dest, len(src), len(stripped)))
         return
 
+    # The compact uploader and fallback WiFi form share the same compressor.
+    static_lines = ["// Auto-generated by scripts/generate_webconfig_html.py.",
+                    "// Google Zopfli %s (gzip, %d iterations)." %
+                    (ZOPFLI_VERSION, MAXIMUM_ITERATIONS),
+                    "#pragma once", "#include <stdint.h>", "#include <pgmspace.h>"]
+    for page in sorted(Path("webui/pages").glob("*.html")):
+        symbol = "MESH_HTML_" + page.stem.upper()
+        compressed = compress_page(page.read_bytes())
+        static_lines.append("const uint32_t %s_LEN = %d;" % (symbol, len(compressed)))
+        append_byte_array(static_lines, symbol, compressed)
+    static_header = Path("src/helpers/esp32/StaticHtml.h")
+    static_rendered = "\n".join(static_lines) + "\n"
+    if not static_header.exists() or static_header.read_text() != static_rendered:
+        static_header.write_text(static_rendered)
+
     src, stripped = shipped_page()
     raw = stripped.encode("utf-8")
     src_hash = content_hash(raw)
 
-    br = compress_html(raw)
-    version = hashlib.sha256(br).hexdigest()[:16]
+    gz = compress_page(raw)
+    version = hashlib.sha256(gz).hexdigest()[:16]
     loader = ("<!doctype html><meta charset=utf-8><meta name=viewport "
               "content=\"width=device-width,initial-scale=1\"><title>MeshCore "
               "WebConfig</title><body>Loading MeshCore WebConfig&hellip;<script>"
@@ -105,27 +145,30 @@ def main():
               "document.write(p);document.close()})().catch(e=>"
               "document.body.textContent=\"Unable to load WebConfig: \"+e+"
               "\". Reload to retry.\")</script>" % version)
-    loader_bytes = loader.encode("utf-8")
+    loader_bytes = compress_page(loader.encode("utf-8"))
+    success_bytes = compress_page(b"<HTML><HEAD><TITLE>Success</TITLE></HEAD><BODY>Success</BODY></HTML>")
     etag = hashlib.sha256(loader_bytes).hexdigest()[:16]
 
     lines = []
     lines.append("// Auto-generated by scripts/generate_webconfig_html.py from %s" % SOURCE.replace(os.sep, "/"))
     lines.append("// DO NOT EDIT - edit webui/index.html instead.")
-    lines.append("// Compressed with Google Brotli %s (quality %d, lgwin %d)." %
-                 (BROTLI_VERSION, QUALITY, LGWIN))
+    lines.append("// Compressed with Google Zopfli %s (gzip, %d iterations)." %
+                 (ZOPFLI_VERSION, MAXIMUM_ITERATIONS))
     lines.append("%s%s" % (HASH_MARKER, src_hash))
     lines.append("#pragma once")
     lines.append("#include <stdint.h>")
     lines.append("#include <pgmspace.h>")
     lines.append("")
-    lines.append("const uint32_t WEBCONFIG_HTML_RAW_LEN = %d;" % len(raw))
-    lines.append("const uint32_t WEBCONFIG_HTML_BR_LEN = %d;" % len(br))
+    lines.append("const uint32_t WEBCONFIG_HTML_GZ_LEN = %d;" % len(gz))
     lines.append('const char WEBCONFIG_HTML_ETAG[] = "\\"%s\\"";' % etag)
     lines.append('const char WEBCONFIG_HTML_VERSION[] = "%s";' % version)
     lines.append("const uint16_t WEBCONFIG_HTML_LOADER_LEN = %d;" % len(loader_bytes))
+    lines.append("const uint16_t WEBCONFIG_HTML_SUCCESS_LEN = %d;" % len(success_bytes))
     append_byte_array(lines, "WEBCONFIG_HTML_LOADER", loader_bytes)
     lines.append("")
-    append_byte_array(lines, "WEBCONFIG_HTML_BR", br)
+    append_byte_array(lines, "WEBCONFIG_HTML_GZ", gz)
+    lines.append("")
+    append_byte_array(lines, "WEBCONFIG_HTML_SUCCESS", success_bytes)
     lines.append("")
 
     rendered = "\n".join(lines)
@@ -140,8 +183,8 @@ def main():
     with open(OUTPUT, "w", encoding="utf-8", newline="") as f:
         f.write(rendered)
 
-    status("%s -> %s (%d bytes source, %d stripped, %d Brotli)"
-           % (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(br)))
+    status("%s -> %s (%d bytes source, %d stripped, %d gzip)"
+           % (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(gz)))
 
 
 # This script is attached only to esp32_base. Generate unconditionally here:

scripts/zopfli_compress.py

@@ -29,8 +29,8 @@ if zopfli.__version__ != ZOPFLI_VERSION:
     )
 
 # Above zlib's maximum level 9: Zopfli refines the LZ77 model repeatedly.
-# Fifteen iterations is the upstream binding's practical default for assets
-# of this size and keeps CI/build times reasonable.
+# Fifteen iterations remains the practical default for raw OTA streams.
+# Firmware HTML gzip assets use the 1,000-iteration setting below.
 ITERATIONS = 15
 # Zopfli has no finite algorithmic iteration ceiling.  Its ``--i1000``
 # convention is the maximum-effort setting used for host-cached mOTA transfer
@@ -49,9 +49,9 @@ def _options(numiterations: int = ITERATIONS) -> dict[str, int | bool]:
     }
 
 
-def gzip_compress(data: bytes) -> bytes:
+def gzip_compress(data: bytes, *, numiterations: int = MAXIMUM_ITERATIONS) -> bytes:
     """Compress *data* as deterministic RFC 1952 gzip using Google Zopfli."""
-    encoded = bytearray(_gzip.compress(data, **_options()))
+    encoded = bytearray(_gzip.compress(data, **_options(numiterations)))
     # The gzip OS byte is informational. Normalize it to RFC 1952's unknown
     # value so a checked-in asset is identical across operating systems.
     if len(encoded) < 10 or encoded[:3] != b"\x1f\x8b\x08":