from __future__ import annotations import argparse import hashlib import json import shutil import subprocess import sys import tempfile from datetime import datetime, timezone from pathlib import Path from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo PACKAGE_SCHEMA_VERSION = 1 ARCHIVE_LAYOUT = "trailmate-pack-v1" FIXED_ZIP_DT = (2026, 1, 1, 0, 0, 0) TOP_LEVEL_PACKAGE_FILES = ("package.ini", "DESCRIPTION.txt", "README.md") PAYLOAD_GROUPS = ("fonts", "locales", "ime") PACKAGE_EXCLUDE_NAMES = {".gitignore", "build.ini", "charset.txt"} def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Build installable locale-pack zip archives and a package catalog for GitHub Pages." ) parser.add_argument("--pack-root", default="packs", help="Repository pack source root") parser.add_argument("--site-root", default="site", help="Pages site root") parser.add_argument("--node-exe", default="node", help="Node.js executable for font generation") parser.add_argument( "--npm-exe", default=None, help="Optional npm executable override for font generation", ) parser.add_argument( "--catalog-path", default=None, help="Optional explicit output path for the generated catalog JSON", ) return parser.parse_args() def clear_directory_contents(path: Path) -> None: path.mkdir(parents=True, exist_ok=True) for child in path.iterdir(): if child.is_dir(): shutil.rmtree(child) else: child.unlink() def parse_key_value_file(path: Path) -> dict[str, str]: data: dict[str, str] = {} for raw_line in path.read_text(encoding="utf-8").splitlines(): line = raw_line.strip() if not line or line.startswith("#") or line.startswith(";"): continue if "=" not in line: continue key, value = line.split("=", 1) key = key.strip().lower() value = value.strip() if key: data[key] = value return data def split_csv(value: str | None) -> list[str]: if not value: return [] return [item.strip() for item in value.split(",") if item.strip()] def parse_int(value: str | None, default: int = 0) -> int: if value is None or value == "": return default return int(value, 10) def parse_bool(value: str | None, default: bool = False) -> bool: if value is None: return default normalized = value.strip().lower() if normalized in {"1", "true", "yes", "on"}: return True if normalized in {"0", "false", "no", "off"}: return False return default def resolve_build_path(spec: str, stage_font_dir: Path, stage_bundle_dir: Path, repo_root: Path) -> Path: candidate = Path(spec) if candidate.is_absolute(): return candidate for base in (stage_font_dir, stage_bundle_dir, repo_root): resolved = (base / candidate).resolve() if resolved.exists(): return resolved return (repo_root / candidate).resolve() def build_font_charset_if_configured( stage_font_dir: Path, stage_bundle_dir: Path, repo_root: Path, ) -> bool: build_path = stage_font_dir / "build.ini" if not build_path.exists(): return False build = parse_key_value_file(build_path) include_manifests = parse_bool(build.get("include_manifests"), default=True) include_locales = parse_bool(build.get("include_locales"), default=True) extra_chars = split_csv(build.get("extra_chars_file")) seed_charsets = split_csv(build.get("seed_charset_file")) exclude_charsets = split_csv(build.get("exclude_charset_file")) ranked_chars_file = build.get("ranked_chars_file", "") ranked_char_limit = parse_int(build.get("ranked_char_limit"), 0) uses_generator = ( not include_manifests or not include_locales or bool(extra_chars) or bool(seed_charsets) or bool(exclude_charsets) or bool(ranked_chars_file) ) if not uses_generator: return False generator = repo_root / "tools" / "build_locale_pack_charset.py" if not generator.exists(): raise FileNotFoundError(f"Missing charset builder script: {generator}") command = [ sys.executable, str(generator), "--pack-root", str(stage_bundle_dir.resolve()), "--font-pack-id", stage_font_dir.name, ] if not include_manifests: command.append("--no-include-manifests") if not include_locales: command.append("--no-include-locales") for spec in extra_chars: command.extend([ "--extra-chars-file", str(resolve_build_path(spec, stage_font_dir, stage_bundle_dir, repo_root)), ]) for spec in seed_charsets: command.extend([ "--seed-charset-file", str(resolve_build_path(spec, stage_font_dir, stage_bundle_dir, repo_root)), ]) for spec in exclude_charsets: command.extend([ "--exclude-charset-file", str(resolve_build_path(spec, stage_font_dir, stage_bundle_dir, repo_root)), ]) if ranked_chars_file: command.extend([ "--ranked-chars-file", str(resolve_build_path(ranked_chars_file, stage_font_dir, stage_bundle_dir, repo_root)), ]) if ranked_char_limit > 0: command.extend(["--ranked-char-limit", str(ranked_char_limit)]) subprocess.run(command, check=True) return True def build_font_if_missing( stage_font_dir: Path, repo_root: Path, node_exe: str, npm_exe: str | None, force_rebuild: bool = False, ) -> None: build_path = stage_font_dir / "build.ini" font_bin_path = stage_font_dir / "font.bin" if force_rebuild and font_bin_path.exists(): font_bin_path.unlink() if font_bin_path.exists(): return if not build_path.exists(): raise FileNotFoundError(f"Missing font.bin and build.ini for {stage_font_dir}") build = parse_key_value_file(build_path) font_source = build.get("font") if not font_source: raise ValueError(f"build.ini missing font=... in {build_path}") charset_file = stage_font_dir / "charset.txt" if not charset_file.exists(): raise FileNotFoundError(f"Missing charset.txt for generated font: {charset_file}") generator = repo_root / "tools" / "generate_binfont_with_lv_font_conv.py" if not generator.exists(): raise FileNotFoundError(f"Missing generator script: {generator}") command = [ sys.executable, str(generator), "--font", str((repo_root / font_source).resolve()), "--charset-file", str(charset_file.resolve()), "--output", str(font_bin_path.resolve()), "--size", str(parse_int(build.get("size"), 16)), "--bpp", str(parse_int(build.get("bpp"), 2)), "--node-exe", node_exe, ] if npm_exe: command.extend(["--npm-exe", npm_exe]) if parse_bool(build.get("no_compress"), default=False): command.append("--no-compress") if parse_bool(build.get("no_prefilter"), default=False): command.append("--no-prefilter") if parse_bool(build.get("no_kerning"), default=False): command.append("--no-kerning") subprocess.run(command, check=True) def stage_bundle( source_bundle_dir: Path, repo_root: Path, node_exe: str, npm_exe: str | None, ) -> Path: temp_dir = Path(tempfile.mkdtemp(prefix=f"trailmate-pack-{source_bundle_dir.name}-")) stage_bundle_dir = temp_dir / source_bundle_dir.name shutil.copytree( source_bundle_dir, stage_bundle_dir, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), ) fonts_root = stage_bundle_dir / "fonts" if fonts_root.is_dir(): for font_dir in sorted(path for path in fonts_root.iterdir() if path.is_dir()): force_rebuild = build_font_charset_if_configured(font_dir, stage_bundle_dir, repo_root) build_font_if_missing(font_dir, repo_root, node_exe, npm_exe, force_rebuild=force_rebuild) return stage_bundle_dir def iter_archive_sources(stage_bundle_dir: Path) -> list[tuple[Path, str]]: files: list[tuple[Path, str]] = [] for name in TOP_LEVEL_PACKAGE_FILES: source = stage_bundle_dir / name if source.is_file(): files.append((source, name)) for group in PAYLOAD_GROUPS: group_dir = stage_bundle_dir / group if not group_dir.is_dir(): continue for source in sorted(path for path in group_dir.rglob("*") if path.is_file()): if source.name in PACKAGE_EXCLUDE_NAMES: continue relative = source.relative_to(stage_bundle_dir).as_posix() files.append((source, f"payload/{relative}")) return files def write_deterministic_zip(files: list[tuple[Path, str]], destination: Path) -> None: destination.parent.mkdir(parents=True, exist_ok=True) with ZipFile(destination, "w") as archive: for source, archive_name in sorted(files, key=lambda item: item[1]): data = source.read_bytes() info = ZipInfo(filename=archive_name, date_time=FIXED_ZIP_DT) info.compress_type = ZIP_DEFLATED info.external_attr = 0o100644 << 16 archive.writestr(info, data) def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def read_package_text(stage_bundle_dir: Path, relative_path: str | None) -> str: if not relative_path: return "" path = stage_bundle_dir / relative_path if not path.is_file(): raise FileNotFoundError(f"Missing package text file: {path}") return path.read_text(encoding="utf-8").strip() def collect_font_records(stage_bundle_dir: Path) -> list[dict[str, object]]: records: list[dict[str, object]] = [] fonts_root = stage_bundle_dir / "fonts" if not fonts_root.is_dir(): return records for manifest_path in sorted(fonts_root.glob("*/manifest.ini")): manifest = parse_key_value_file(manifest_path) records.append( { "id": manifest.get("id", manifest_path.parent.name), "display_name": manifest.get("display_name", manifest_path.parent.name), "usage": manifest.get("usage", "both"), "estimated_ram_bytes": parse_int(manifest.get("estimated_ram_bytes"), 0), } ) return records def collect_locale_records(stage_bundle_dir: Path) -> list[dict[str, object]]: records: list[dict[str, object]] = [] locales_root = stage_bundle_dir / "locales" if not locales_root.is_dir(): return records for manifest_path in sorted(locales_root.glob("*/manifest.ini")): manifest = parse_key_value_file(manifest_path) records.append( { "id": manifest.get("id", manifest_path.parent.name), "display_name": manifest.get("display_name", manifest_path.parent.name), "native_name": manifest.get("native_name", manifest.get("display_name", manifest_path.parent.name)), "ui_font_pack_id": manifest.get("ui_font_pack", ""), "content_font_pack_id": manifest.get( "content_font_pack", manifest.get("ui_font_pack", "") ), "translation_status": manifest.get("translation_status", "release"), "direction": manifest.get("direction", "ltr"), "preferred_content_supplement_pack_ids": split_csv( manifest.get("preferred_content_supplement_packs", manifest.get("content_supplement_packs", "")) ), "ime_pack_id": manifest.get("ime_pack", ""), } ) return records def collect_ime_records(stage_bundle_dir: Path) -> list[dict[str, object]]: records: list[dict[str, object]] = [] ime_root = stage_bundle_dir / "ime" if not ime_root.is_dir(): return records for manifest_path in sorted(ime_root.glob("*/manifest.ini")): manifest = parse_key_value_file(manifest_path) records.append( { "id": manifest.get("id", manifest_path.parent.name), "display_name": manifest.get("display_name", manifest_path.parent.name), "backend": manifest.get("backend", "none"), } ) return records def build_package_entry( stage_bundle_dir: Path, package_manifest: dict[str, str], archive_relative_path: str, archive_path: Path, ) -> dict[str, object]: fonts = collect_font_records(stage_bundle_dir) locales = collect_locale_records(stage_bundle_dir) ime = collect_ime_records(stage_bundle_dir) description_text = read_package_text(stage_bundle_dir, package_manifest.get("description")) readme_path = package_manifest.get("readme") readme_relative_path = readme_path if readme_path and (stage_bundle_dir / readme_path).is_file() else "" return { "id": package_manifest["id"], "package_type": package_manifest.get("package_type", "locale-bundle"), "version": package_manifest["version"], "display_name": package_manifest.get("display_name", package_manifest["id"]), "summary": package_manifest.get("summary", ""), "description": description_text, "author": package_manifest.get("author", "Trail Mate"), "homepage": package_manifest.get("homepage", ""), "min_firmware_version": package_manifest.get("min_firmware_version", ""), "supported_memory_profiles": split_csv(package_manifest.get("supported_memory_profiles")), "tags": split_csv(package_manifest.get("tags")), "provides": { "locales": locales, "fonts": fonts, "ime": ime, }, "runtime": { "estimated_unique_font_ram_bytes": sum( {record["id"]: int(record["estimated_ram_bytes"]) for record in fonts}.values() ), "locale_count": len(locales), "font_count": len(fonts), "ime_count": len(ime), }, "archive": { "layout": ARCHIVE_LAYOUT, "path": archive_relative_path.replace("\\", "/"), "size_bytes": archive_path.stat().st_size, "sha256": sha256_file(archive_path), }, "files": { "package_manifest": "package.ini", "description": package_manifest.get("description", ""), "readme": readme_relative_path, }, } def discover_source_bundles(pack_root: Path) -> list[Path]: bundles: list[Path] = [] for candidate in sorted(path for path in pack_root.iterdir() if path.is_dir()): if (candidate / "package.ini").is_file(): bundles.append(candidate) return bundles def build_pack_repository( pack_root: Path, site_root: Path, node_exe: str = "node", npm_exe: str | None = None, catalog_path: Path | None = None, ) -> dict[str, object]: pack_root = pack_root.resolve() site_root = site_root.resolve() repo_root = pack_root.parent.resolve() assets_dir = site_root / "assets" / "packs" clear_directory_contents(assets_dir) entries: list[dict[str, object]] = [] temp_roots: list[Path] = [] try: for bundle_dir in discover_source_bundles(pack_root): package_manifest = parse_key_value_file(bundle_dir / "package.ini") if package_manifest.get("kind") != "package": continue if "id" not in package_manifest or "version" not in package_manifest: raise ValueError(f"package.ini missing id/version in {bundle_dir}") stage_bundle_dir = stage_bundle(bundle_dir, repo_root, node_exe, npm_exe) temp_roots.append(stage_bundle_dir.parent) archive_name = f"{package_manifest['id']}-{package_manifest['version']}.zip" archive_path = assets_dir / archive_name write_deterministic_zip(iter_archive_sources(stage_bundle_dir), archive_path) archive_relative_path = archive_path.relative_to(site_root).as_posix() entries.append( build_package_entry( stage_bundle_dir, package_manifest, archive_relative_path, archive_path, ) ) finally: for temp_root in temp_roots: shutil.rmtree(temp_root, ignore_errors=True) entries.sort(key=lambda entry: str(entry["display_name"]).lower()) payload = { "schema_version": PACKAGE_SCHEMA_VERSION, "archive_layout": ARCHIVE_LAYOUT, "generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "packages": entries, } output_path = catalog_path or (site_root / "data" / "packs.json") output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") return payload def main() -> int: args = parse_args() build_pack_repository( pack_root=Path(args.pack_root), site_root=Path(args.site_root), node_exe=args.node_exe, npm_exe=args.npm_exe, catalog_path=Path(args.catalog_path) if args.catalog_path else None, ) return 0 if __name__ == "__main__": raise SystemExit(main())