diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index f4e66f00f..81270ac91 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -276,6 +276,7 @@ jobs: - name: Verify official web console framing and power-saving idle sessions run: | + python3 -B test/test_cpp_source.py -v python3 -B test/test_official_web_console.py -v python3 -B test/test_hil_linux_web_serial.py -v python3 -B test/test_hil_slowfi_guard.py -v diff --git a/test/cpp_source.py b/test/cpp_source.py new file mode 100644 index 000000000..98f03733f --- /dev/null +++ b/test/cpp_source.py @@ -0,0 +1,198 @@ +"""Extract original C++ compound bodies without choosing a build configuration. + +This is a lexical test helper, not a C++ parser or preprocessor. Conditional +alternatives must agree on brace depth at their join. All alternatives and +directives remain in the returned source for the real compiler to select. +""" + +import re + + +_RAW = re.compile(r'(?:u8|u|U|L)?R"([^\s()\\]{0,16})\(') +_DIRECTIVE = re.compile(r"#\s*([A-Za-z_]\w*)") + + +def _quoted_end(text, start): + quote = text[start] + i = start + 1 + while i < len(text): + if text[i] == "\\": + i += 3 if text.startswith("\r\n", i + 1) else 2 + elif text[i] == quote: + return i + 1 + elif text[i] in "\r\n": + raise AssertionError("unclosed C++ quoted literal") + else: + i += 1 + raise AssertionError("unclosed C++ quoted literal") + + +def _number_separator(text, i): + if i + 1 == len(text) or not text[i + 1].isalnum(): + return False + start = i + while start and (text[start - 1].isalnum() or text[start - 1] in "_'"): + start -= 1 + return start < i and text[start].isdigit() + + +def _line_end(text, start): + end = text.find("\n", start) + while end >= 0 and text[max(0, end - 2):end].endswith(("\\", "\\\r")): + end = text.find("\n", end + 1) + return len(text) if end < 0 else end + 1 + + +def _block_comment_end(text, start): + end = text.find("*/", start + 2) + if end < 0: + raise AssertionError("unclosed C++ block comment") + return end + 2 + + +def _raw_end(text, start): + raw = _RAW.match(text, start) if text[start] in "uULR" else None + if raw: + closing = ")" + raw.group(1) + '"' + end = text.find(closing, raw.end()) + if end < 0: + raise AssertionError("unclosed C++ raw string literal") + return end + len(closing) + if text.startswith('R"', start): + raise AssertionError("invalid C++ raw string delimiter") + return None + + +def _directive_end(text, start): + # Comments and raw literals may span physical lines inside a directive; + # only an unescaped newline outside those tokens ends its logical line. + i = start + 1 + while i < len(text): + if text.startswith("\\\r\n", i): + i += 3 + elif text.startswith("\\\n", i): + i += 2 + elif text[i] in "\r\n": + return i + (2 if text.startswith("\r\n", i) else 1) + elif text.startswith("//", i): + return _line_end(text, i + 2) + elif text.startswith("/*", i): + i = _block_comment_end(text, i) + else: + raw_end = _raw_end(text, i) + if raw_end is not None: + i = raw_end + elif text[i] == '"' or (text[i] == "'" and not _number_separator(text, i)): + i = _quoted_end(text, i) + else: + i += 1 + return i + + +def _events(text): + i = 0 + line_start = True + while i < len(text): + start = i + if text.startswith("\\\n", i) or text.startswith("\\\r\n", i): + i += 3 if text.startswith("\\\r\n", i) else 2 + yield "ignored", start, i, None + continue + if text[i].isspace(): + if text[i] in "\r\n": + line_start = True + i += 1 + continue + if text.startswith("//", i): + i = _line_end(text, i + 2) + line_start = True + yield "ignored", start, i, None + continue + if text.startswith("/*", i): + i = _block_comment_end(text, i) + if "\n" in text[start:i] or "\r" in text[start:i]: + line_start = True + yield "ignored", start, i, None + continue + if text[i] == "#" and line_start: + i = _directive_end(text, i) + logical = re.sub(r"\\\r?\n", "", text[start:i]) + logical = re.sub(r"/\*.*?\*/", " ", logical, flags=re.DOTALL) + match = _DIRECTIVE.match(logical) + yield "directive", start, i, match.group(1) if match else "" + line_start = True + continue + raw_end = _raw_end(text, i) + if raw_end is not None: + i = raw_end + yield "literal", start, i, None + elif text[i] == '"' or (text[i] == "'" and not _number_separator(text, i)): + i = _quoted_end(text, i) + yield "literal", start, i, None + else: + i += 1 + yield "brace" if text[start] in "{}" else "code", start, i, text[start] + line_start = False + + +def body(text, signature): + """Return the original signature and complete compound body. + + Preserve the legacy helper's ValueError for a missing signature and + AssertionError for incomplete or ambiguous bodies. Signatures in comments, + literals and macro definitions do not select a body. + """ + if not signature: + raise ValueError("empty C++ signature") + start = text.find(signature) + events = iter(_events(text)) + for kind, begin, end, value in events: + if start < 0: + raise ValueError("C++ signature not found: " + signature) + while begin <= start < end and kind not in ("code", "brace"): + start = text.find(signature, end) + if begin <= start < end: + break + else: + raise ValueError("C++ signature not found: " + signature) + + for kind, begin, end, value in events: + if kind == "brace" and value == "{": + break + else: + raise AssertionError("missing C++ body opening brace") + + depth = 1 + branches = [] + for kind, begin, end, value in events: + if kind == "directive": + if value in ("if", "ifdef", "ifndef"): + branches.append({"entry": depth, "ends": [], "else": False}) + elif value in ("elif", "else"): + if not branches or branches[-1]["else"]: + raise AssertionError("unexpected C++ conditional alternative") + frame = branches[-1] + frame["ends"].append(depth) + frame["else"] = value == "else" + depth = frame["entry"] + elif value == "endif": + if not branches: + raise AssertionError("unexpected C++ conditional endif") + frame = branches.pop() + frame["ends"].append(depth) + if not frame["else"]: + frame["ends"].append(frame["entry"]) + if len(set(frame["ends"])) != 1: + raise AssertionError("C++ conditional alternatives have different brace depths") + depth = frame["ends"][0] + if depth == 0 and not branches: + return text[start:end] + elif depth == 0 and kind != "ignored": + raise AssertionError("code after a conditional C++ body closing brace") + elif kind == "brace": + depth += 1 if value == "{" else -1 + if depth < 0: + raise AssertionError("unexpected C++ closing brace") + if depth == 0 and not branches: + return text[start:end] + raise AssertionError("unclosed C++ body or conditional") diff --git a/test/test_cpp_source.py b/test/test_cpp_source.py new file mode 100644 index 000000000..444155a78 --- /dev/null +++ b/test/test_cpp_source.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Qualify the shared C++ extractor against real and conditional source.""" + +from pathlib import Path +import subprocess +import tempfile +import unittest + +from cpp_source import body + + +ROOT = Path(__file__).resolve().parents[1] + + +class CppSourceTests(unittest.TestCase): + def test_stops_at_the_actual_body_boundary(self): + function = "void target() { if (true) { call(); } }" + self.assertEqual(body(function + "\nvoid next() {}", "void target()"), function) + + def test_preserves_legacy_if_and_class_extraction(self): + source = 'if (strcmp(config, "powersaving on") == 0) { set(true); } else { set(false); }' + self.assertEqual(body(source, 'if (strcmp(config, "powersaving on") == 0'), source.split(" else")[0]) + self.assertEqual(body("class Stream { int read() { return 0; } };", "class Stream"), + "class Stream { int read() { return 0; } }") + + def test_ignores_comments_strings_chars_and_raw_strings(self): + function = r'''void target(const char* value = "{") { + // }}} #else + /* { #endif + } */ + const char* text = "escaped \" quote } {"; + char close = '}'; char escaped = '\''; + const char* raw = u8R"mark({ } // /* " +#else +void target() { }} +)mark"; + unsigned number = 0xff'aa; unsigned decimal = 1'000'000; +}''' + self.assertEqual(body(function + "\nvoid next() {}", "void target("), function) + + def test_signature_in_comment_literal_or_macro_does_not_select_a_body(self): + prefix = '''// void target() { } +/* void target() { } */ +const char* example = "void target() { }"; +#define EXAMPLE void target() { } +''' + actual = "void target() { work(); }" + self.assertEqual(body(prefix + actual, "void target()"), actual) + + def test_escaped_newlines_keep_comments_and_literals_lexical(self): + function = 'void target() {\n// } \\\n} still a comment\nconst char* x = "} \\\r\n {";\n}' + self.assertEqual(body(function + "\nvoid next() {}", "void target()"), function) + + def test_macro_braces_and_continuations_are_not_body_braces(self): + function = '''void target() { +#define OPEN { \\ + { } +#define CLOSE } + work(); +}''' + self.assertEqual(body(function + "\nvoid next() {}", "void target()"), function) + + def test_nested_conditional_openings_share_the_common_close(self): + function = '''void target() { +/* prefix */ #if OUTER +#if A + if (first) { +#elif B + if (second) { +#else + if (third) { +#endif +#else + if (fallback) { +#endif + work(); + } +}''' + self.assertEqual(body(function + "\nvoid next() {}", "void target()"), function) + + def test_multiline_macro_comments_and_raw_literals_hide_their_braces(self): + function = '''void target() { +#define COMMENT /* comment +} } #endif +*/ 1 +#define LITERAL R"tag( +} } #else +)tag" +# /* directive comment +} } */ if ENABLED + if (first) { +#else + if (second) { +#endif + work(); + } +}''' + self.assertEqual(body(function + "\nvoid next() {}", "void target()"), function) + + def test_conditional_closings_retain_the_endif_boundary(self): + function = "void target() {\n#if A\n}\n#else\n}\n#endif\n" + self.assertEqual(body(function + "void next() {}", "void target()"), function) + + def test_optional_balanced_blocks_join_the_implicit_else(self): + function = "void target() {\n#if A\n if (first) { work(); }\n#endif\n}" + self.assertEqual(body(function + "\nvoid next() {}", "void target()"), function) + + def test_malformed_or_ambiguous_sources_fail_explicitly(self): + malformed = ( + "void target() {", + "void target() { /* }", + "void target() {\n#define VALUE /* }\n}", + 'void target() {\n#define VALUE R"tag(}\n}', + 'void target() { "}', + "void target() { '}", + 'void target() { R"tag(})other"; }', + 'void target() { R"bad delimiter(})bad delimiter"; }', + "void target() {\n#if A\n}", + "void target() {\n#else\n}", + "void target() {\n#endif\n}", + "void target() {\n#if A\n#else\n#else\n#endif\n}", + "void target() {\n#if A\n#else\n#elif B\n#endif\n}", + "void target() {\n#if A\nif (a) {\n#else\n#endif\n}}", + "void target() {\n#if A\nif (a) {\n#endif\n}}", + "void target() {\n#if A\n}\nvoid hidden() {}\n#else\n}\n#endif", + ) + for source in malformed: + with self.subTest(source=source), self.assertRaises(AssertionError): + body(source, "void target()") + + def test_missing_signature_and_opening_brace_keep_the_api_errors(self): + with self.assertRaises(ValueError): + body("void other() {}", "void target()") + with self.assertRaises(ValueError): + body("void target() {}", "") + with self.assertRaises(AssertionError): + body("void target();", "void target()") + + def test_actual_repeater_command_pump_with_both_ethernet_alternatives(self): + source = (ROOT / "examples/simple_repeater/main.cpp").read_text() + signature = "static void __attribute__((noinline)) serviceCommandInterfaces()" + extracted = body(source, signature) + expected = source[source.index(signature):source.index("\nstatic bool usbLoggingRecoverySafe")].rstrip() + self.assertEqual(extracted, expected) + self.assertIn("!ethernet_get_enabled()", extracted) + self.assertIn("!ethernet_client.connected()", extracted) + self.assertNotIn("usbLoggingRecoverySafe", extracted) + + def test_both_real_conditional_forms_compile_without_firmware_changes(self): + source = '''int target(bool condition) { + int value = 0; + (void)condition; +#ifdef ETHERNET_ENABLED +#if defined(RAK4631_COMBINED_ETHERNET) + if (condition) { +#else + if (!condition) { +#endif + value = 3; + } +#endif + return value; +} +void must_not_be_extracted() { missing_symbol(); } +''' + extracted = body(source, "int target(") + with tempfile.TemporaryDirectory() as directory: + cpp = Path(directory) / "conditional.cpp" + cpp.write_text("#include \n" + extracted + r''' +int main() { +#if !defined(ETHERNET_ENABLED) + assert(target(false) == 0 && target(true) == 0); +#elif defined(RAK4631_COMBINED_ETHERNET) + assert(target(false) == 0 && target(true) == 3); +#else + assert(target(false) == 3 && target(true) == 0); +#endif +} +''') + for defines in ([], ["-DETHERNET_ENABLED"], + ["-DETHERNET_ENABLED", "-DRAK4631_COMBINED_ETHERNET"]): + with self.subTest(defines=defines): + binary = Path(directory) / "conditional" + compile_result = subprocess.run( + ["g++", "-std=c++17", "-Wall", "-Wextra", "-Werror", *defines, + str(cpp), "-o", str(binary)], capture_output=True, text=True, timeout=30) + self.assertEqual(compile_result.returncode, 0, compile_result.stderr) + run = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(run.returncode, 0, run.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_hwcdc_tx_backport.py b/test/test_hwcdc_tx_backport.py index 705474e7c..0c49f5f6f 100644 --- a/test/test_hwcdc_tx_backport.py +++ b/test/test_hwcdc_tx_backport.py @@ -5,6 +5,7 @@ from pathlib import Path import subprocess import tempfile import unittest +from cpp_source import body ROOT=Path(__file__).resolve().parents[1] CORE=ROOT/'test/fixtures/hwcdc_tx_backport' @@ -16,15 +17,6 @@ exec(compile(Path(spec.origin).read_text(),spec.origin,'exec'),module.__dict__) RAW=(CORE/'HWCDC.cpp').read_text() PATCHED=module.patched_hwcdc_source(RAW) -def body(text,signature): - start=text.index(signature);open_=text.index('{',start);depth=0 - for end in range(open_,len(text)): - if text[end]=='{':depth+=1 - elif text[end]=='}': - depth-=1 - if not depth:return text[start:end+1] - raise AssertionError('unclosed function') - HARNESS=r''' #include #include