fix(ci): support UTF-8 changelog version and release extraction

.github/workflows/ci.yml

@@ -414,7 +414,7 @@ jobs:
           if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
             VERSION="${GITHUB_REF_NAME#v}"
           else
-            VERSION=$(iconv -f UTF-16LE -t UTF-8 CHANGELOG.md | awk -F'[][]' '/^## \[[0-9]/ {print $2; exit}')
+            VERSION=$(python3 scripts/read_changelog.py --version)
           fi
           if [ -z "$VERSION" ]; then
             VERSION="unknown"
@@ -596,7 +596,7 @@ jobs:
           if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
             VERSION="${GITHUB_REF_NAME#v}"
           else
-            VERSION=$(iconv -f UTF-16LE -t UTF-8 CHANGELOG.md | awk -F'[][]' '/^## \[[0-9]/ {print $2; exit}')
+            VERSION=$(python3 scripts/read_changelog.py --version)
           fi
           if [ -z "$VERSION" ]; then
             VERSION="unknown"
@@ -687,7 +687,7 @@ jobs:
           if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
             VERSION="${GITHUB_REF_NAME#v}"
           else
-            VERSION=$(iconv -f UTF-16LE -t UTF-8 CHANGELOG.md | awk -F'[][]' '/^## \[[0-9]/ {print $2; exit}')
+            VERSION=$(python3 scripts/read_changelog.py --version)
           fi
           if [ -z "$VERSION" ]; then
             VERSION="unknown"
@@ -786,7 +786,7 @@ jobs:
       - name: Extract changelog entry
         run: |
           VERSION="${GITHUB_REF_NAME#v}"
-          iconv -f UTF-16LE -t UTF-8 CHANGELOG.md > CHANGELOG.utf8.md
+          python3 scripts/read_changelog.py > CHANGELOG.utf8.md
           awk -v version="$VERSION" '
             $0 ~ "^## \\[" version "\\]" {capture=1}
             capture {

scripts/read_changelog.py

@@ -0,0 +1,27 @@
+"""Read current UTF-8 and historical BOM-marked UTF-16 changelogs for CI."""
+
+import argparse
+from pathlib import Path
+import re
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("changelog", nargs="?", type=Path, default=Path("CHANGELOG.md"))
+    parser.add_argument("--version", action="store_true", help="Print the first version heading only")
+    args = parser.parse_args()
+    raw = args.changelog.read_bytes()
+    text = raw.decode("utf-16" if raw.startswith((b"\xff\xfe", b"\xfe\xff")) else "utf-8-sig")
+    text = text.replace("\r\r\n", "\n").replace("\r\n", "\n")
+    if args.version:
+        match = re.search(r"^## \[([0-9][^\]]*)\]", text, re.MULTILINE)
+        if not match:
+            parser.error("Changelog has no version heading")
+        print(match.group(1).strip())
+    else:
+        sys.stdout.write(text)
+
+
+if __name__ == "__main__":
+    main()

tests/test_firmware_version.py

@@ -4,6 +4,9 @@ import ast
 import os
 from pathlib import Path
 import re
+import subprocess
+import sys
+import tempfile
 import unittest
 from unittest.mock import patch
 
@@ -22,6 +25,29 @@ exec(compile(ast.Module(body=functions, type_ignores=[]), "version-helpers", "ex
 
 
 class FirmwareVersionTest(unittest.TestCase):
+    def test_ci_reads_current_and_historical_changelog_encodings(self):
+        text = "# Changelog\n\n## [1.0.0] - Unreleased\n\nRelease notes.\n\n## [0.1.53-alpha]\nOlder notes.\n"
+        with tempfile.TemporaryDirectory() as folder:
+            source = Path(folder) / "CHANGELOG.md"
+            for encoding in ("utf-8", "utf-8-sig", "utf-16"):
+                source.write_bytes(text.encode(encoding))
+                command = [sys.executable, str(ROOT / "scripts/read_changelog.py"), str(source)]
+                version = subprocess.check_output(command + ["--version"], text=True)
+                self.assertEqual(version.strip(), "1.0.0", encoding)
+                body = subprocess.check_output(command, text=True)
+                self.assertEqual(body, text, encoding)
+                with patch.dict(namespace, {"project_dir": folder}):
+                    self.assertEqual(namespace["extract_version_from_changelog"](), version.strip())
+
+    def test_ci_rejects_missing_version_before_building(self):
+        with tempfile.TemporaryDirectory() as folder:
+            source = Path(folder) / "CHANGELOG.md"
+            source.write_text("# No version heading\n", encoding="utf-8")
+            result = subprocess.run([sys.executable, str(ROOT / "scripts/read_changelog.py"), str(source), "--version"],
+                                    capture_output=True, text=True)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("no version heading", result.stderr)
+
     def test_local_build_uses_changelog(self):
         with patch.dict(os.environ, {}, clear=True):
             self.assertEqual(namespace["resolve_project_version"](),