diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 890aab3479..919935464d 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -8,7 +8,7 @@ on: type: choice required: true default: all - options: [all, linux-static, android, windows, x86_64-darwin] + options: [all, linux-static, android, windows, x86_64-darwin, x86_64-darwin-libs] push: branches: [master, stable, build/stable-ghc-9.14] paths: @@ -52,7 +52,7 @@ jobs: - name: Select existing build groups id: select env: - # PRs and pushes always retain all seven build groups. + # PRs and pushes always retain all eight build groups. NIX_CI_GROUP: ${{ github.event_name == 'workflow_dispatch' && inputs.group || 'all' }} shell: bash run: | @@ -115,7 +115,7 @@ jobs: run: nix build .#checks.x86_64-linux.ghc-elf-comdat-functions --no-link -L --max-jobs 1 --timeout "$NIX_BUILD_TIMEOUT" - name: Check Intel Darwin build policies - if: ${{ matrix.system == 'x86_64-darwin' && matrix.target == 'all' }} + if: ${{ matrix.system == 'x86_64-darwin' }} timeout-minutes: 1 run: | python3 -B scripts/nix/test-select-intel-setup.py @@ -150,7 +150,7 @@ jobs: run: make nix-build 2>&1 | tee nix-build.log - name: Check native Intel Darwin GHC info-table placement - if: ${{ matrix.system == 'x86_64-darwin' && matrix.target == 'all' }} + if: ${{ matrix.system == 'x86_64-darwin' }} timeout-minutes: 10 run: nix build .#checks.x86_64-darwin.ghc-macho-info-tables --no-link -L --max-jobs 1 --timeout 300 @@ -213,7 +213,7 @@ jobs: - name: Capture Intel Darwin ghc-boot Setup failure id: intel_setup_trace - if: ${{ failure() && matrix.system == 'x86_64-darwin' && matrix.target == 'all' && steps.haskell_build.outcome == 'failure' }} + if: ${{ failure() && matrix.system == 'x86_64-darwin' && steps.haskell_build.outcome == 'failure' }} timeout-minutes: 120 shell: bash run: | diff --git a/scripts/nix/README.md b/scripts/nix/README.md index e0f9526a8c..6076291097 100644 --- a/scripts/nix/README.md +++ b/scripts/nix/README.md @@ -294,17 +294,26 @@ make nix-native NIX_ARGS=--dry-run ``` `.github/workflows/nix.yml` runs these same outputs on Linux ARM64/x86_64 and -Darwin ARM64/x86_64, with a six-hour limit per build job. Its seven build jobs cover all 21 +Darwin ARM64/x86_64, with a six-hour limit per build job. Its eight build jobs cover all 21 outputs once: x86_64 Linux runs `native`, `android`, `windows`, and `linux-static` -separately; ARM64 Linux runs `native`; both Darwin hosts run `all`. A failed +separately; ARM64 Linux runs `native`; ARM64 Darwin runs `all`; Intel Darwin runs +`native` and `darwin-libs` separately. On a cold Intel runner the combined job +finished the native library and CLI only after about five and a half hours, +then hit GitHub's six-hour cap while building mobile dependencies. The split +gives each group its own existing six-hour budget and artifact upload; both use +the same checkout, lock file, and `ghc914-sh`. Intel policy, info-table and +failure diagnostic checks apply to both groups; CLI smoke runs in `native`. A failed native Linux build therefore leaves the cross-build jobs running. PRs, pushes, and default manual runs retain that complete matrix. A manual dispatch can select the existing three `linux-static` outputs with `gh workflow run nix.yml --ref BRANCH -f group=linux-static`, or the four Android outputs with `gh workflow run nix.yml --ref BRANCH -f group=android`, or the two Windows outputs with `gh workflow run nix.yml --ref BRANCH -f group=windows`, or -all four Intel Darwin outputs with -`gh workflow run nix.yml --ref BRANCH -f group=x86_64-darwin`. The registered +all four Intel Darwin outputs in both jobs with +`gh workflow run nix.yml --ref BRANCH -f group=x86_64-darwin`. Select only the two +Intel mobile library outputs with +`gh workflow run nix.yml --ref BRANCH -f group=x86_64-darwin-libs`; its distinct +manual queue can run while an earlier full Intel trial remains active. The registered workflow can run on a branch containing this workflow file; it does not require that file on the default branch. Runs for the same workflow/ref queue without canceling an active run; manual groups have separate queues. Legacy PR build jobs also preserve running diff --git a/scripts/nix/ci-matrix.py b/scripts/nix/ci-matrix.py index 57066f7a29..3f819f2716 100644 --- a/scripts/nix/ci-matrix.py +++ b/scripts/nix/ci-matrix.py @@ -10,17 +10,22 @@ ROWS = ( ("x86_64-linux", "ubuntu-24.04", "linux-static"), ("aarch64-linux", "ubuntu-24.04-arm", "native"), ("aarch64-darwin", "macos-15", "all"), - ("x86_64-darwin", "macos-15-intel", "all"), + # Cold native compiler/application builds left too little of GitHub's + # six-hour job limit for the two mobile libraries. Keep their budgets separate. + ("x86_64-darwin", "macos-15-intel", "native"), + ("x86_64-darwin", "macos-15-intel", "darwin-libs"), ) def select(group): - if group not in ("all", "linux-static", "android", "windows", "x86_64-darwin"): + if group not in ("all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"): raise ValueError(f"unknown Nix CI group: {group}") if group == "all": rows = ROWS elif group == "x86_64-darwin": rows = tuple(row for row in ROWS if row[0] == group) + elif group == "x86_64-darwin-libs": + rows = tuple(row for row in ROWS if row[0] == "x86_64-darwin" and row[2] == "darwin-libs") else: rows = tuple(row for row in ROWS if row[2] == group) return {"include": [dict(system=system, runner=runner, target=target) @@ -29,5 +34,5 @@ def select(group): if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--group", choices=["all", "linux-static", "android", "windows", "x86_64-darwin"], default="all") + parser.add_argument("--group", choices=["all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"], default="all") print(json.dumps(select(parser.parse_args().group), separators=(",", ":"))) diff --git a/scripts/nix/test-ci-matrix.py b/scripts/nix/test-ci-matrix.py index 440c7cd017..b35434fbbe 100644 --- a/scripts/nix/test-ci-matrix.py +++ b/scripts/nix/test-ci-matrix.py @@ -14,21 +14,47 @@ matrix = importlib.util.module_from_spec(spec) spec.loader.exec_module(matrix) +def listed_outputs(system, target): + env = dict(os.environ, NIX_SYSTEM=system, NIX_TARGET=target) + return subprocess.check_output(["make", "--no-print-directory", "nix-list"], + cwd=HERE.parent.parent, env=env, text=True).splitlines() + + def outputs(group): - result = [] - for row in matrix.select(group)["include"]: - env = dict(os.environ, NIX_SYSTEM=row["system"], NIX_TARGET=row["target"]) - result += subprocess.check_output(["make", "--no-print-directory", "nix-list"], - cwd=HERE.parent.parent, env=env, text=True).splitlines() - return result + return [output for row in matrix.select(group)["include"] + for output in listed_outputs(row["system"], row["target"])] + + +def workflow_step(name): + source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text() + match = re.search(r"^ - name: " + re.escape(name) + r"\n(.*?)(?=^ - |\Z)", + source, re.MULTILINE | re.DOTALL) + if match is None: + raise AssertionError(f"missing workflow step: {name}") + return match.group(1) + + +def step_enabled(name, row, failed=False, build_outcome="success"): + match = re.search(r"^ if: \$\{\{ (.*) \}\}$", workflow_step(name), re.MULTILINE) + if match is None: + raise AssertionError(f"missing workflow condition: {name}") + expression = match.group(1).replace("matrix.system", "system").replace("matrix.target", "target") + expression = expression.replace("steps.haskell_build.outcome", "build_outcome") + expression = expression.replace("failure()", "failed").replace("&&", "and").replace("||", "or") + return bool(eval(expression, {"__builtins__": {}}, { + **row, "failed": failed, "build_outcome": build_outcome, + })) class Matrix(unittest.TestCase): def test_default_covers_all_21_outputs_once(self): selected = outputs("all") - self.assertEqual(len(matrix.select("all")["include"]), 7) + self.assertEqual(len(matrix.select("all")["include"]), 8) self.assertEqual(len(selected), 21) self.assertEqual(len(set(selected)), 21) + self.assertEqual(selected, [output for system in [ + "x86_64-linux", "aarch64-linux", "aarch64-darwin", "x86_64-darwin", + ] for output in listed_outputs(system, "all")]) def test_static_dispatch_is_exactly_the_existing_three_outputs(self): self.assertEqual(outputs("linux-static"), [ @@ -51,24 +77,99 @@ class Matrix(unittest.TestCase): "hydraJobs.x86_64-linux.x86_64-windows:lib:simplex-chat", ]) - def test_intel_dispatch_retains_all_four_existing_outputs(self): + def test_intel_dispatch_splits_all_four_existing_outputs_in_order(self): selected = matrix.select("x86_64-darwin") - self.assertEqual(selected, {"include": [dict( - system="x86_64-darwin", runner="macos-15-intel", target="all")]}) - self.assertEqual(outputs("x86_64-darwin"), [ + self.assertEqual(selected, {"include": [ + dict(system="x86_64-darwin", runner="macos-15-intel", target="native"), + dict(system="x86_64-darwin", runner="macos-15-intel", target="darwin-libs"), + ]}) + native = [ "hydraJobs.x86_64-darwin.lib:simplex-chat", "hydraJobs.x86_64-darwin.exe:simplex-chat", + ] + mobile = [ + "hydraJobs.x86_64-darwin.x86_64-darwin-ios:lib:simplex-chat", + "hydraJobs.x86_64-darwin.x86_64-darwin:lib:simplex-chat", + ] + self.assertEqual(listed_outputs("x86_64-darwin", "native"), native) + self.assertEqual(listed_outputs("x86_64-darwin", "darwin-libs"), mobile) + self.assertEqual(outputs("x86_64-darwin"), native + mobile) + cli = subprocess.check_output([ + "python3", str(HERE / "ci-matrix.py"), "--group", "x86_64-darwin"], text=True) + self.assertEqual(json.loads(cli), selected) + + def test_intel_mobile_dispatch_is_only_the_two_existing_library_outputs(self): + selected = matrix.select("x86_64-darwin-libs") + self.assertEqual(selected, {"include": [dict( + system="x86_64-darwin", runner="macos-15-intel", target="darwin-libs")]}) + self.assertEqual(outputs("x86_64-darwin-libs"), [ "hydraJobs.x86_64-darwin.x86_64-darwin-ios:lib:simplex-chat", "hydraJobs.x86_64-darwin.x86_64-darwin:lib:simplex-chat", ]) cli = subprocess.check_output([ - "python3", str(HERE / "ci-matrix.py"), "--group", "x86_64-darwin"], text=True) + "python3", str(HERE / "ci-matrix.py"), "--group", "x86_64-darwin-libs"], text=True) self.assertEqual(json.loads(cli), selected) + def test_workflow_manual_choices_match_the_selector(self): + source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text() + choices = re.search(r"^ options: \[(.*)\]$", source, re.MULTILINE) + self.assertIsNotNone(choices) + expected = ["all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"] + self.assertEqual(choices.group(1).split(", "), expected) + for group in expected: + with self.subTest(group=group): + cli = subprocess.check_output([ + "python3", str(HERE / "ci-matrix.py"), "--group", group], text=True) + self.assertEqual(json.loads(cli), matrix.select(group)) + + def test_intel_policy_and_info_table_checks_cover_both_groups(self): + for row in matrix.select("all")["include"]: + for step in ["Check Intel Darwin build policies", + "Check native Intel Darwin GHC info-table placement"]: + with self.subTest(row=row, step=step): + self.assertEqual(step_enabled(step, row), row["system"] == "x86_64-darwin") + for target in ["native", "darwin-libs"]: + row = dict(system="x86_64-darwin", target=target) + self.assertTrue(step_enabled("Check Intel Darwin build policies", row)) + self.assertTrue(step_enabled("Check native Intel Darwin GHC info-table placement", row)) + + def test_intel_setup_diagnostic_keeps_exact_failure_gate_in_both_groups(self): + for row in matrix.select("all")["include"]: + for failed in [False, True]: + for outcome in ["success", "failure", "cancelled", "skipped"]: + with self.subTest(row=row, failed=failed, outcome=outcome): + self.assertEqual(step_enabled("Capture Intel Darwin ghc-boot Setup failure", row, + failed, outcome), + row["system"] == "x86_64-darwin" and failed and outcome == "failure") + + def test_native_cli_smoke_matches_the_actual_output_group(self): + for row in matrix.select("all")["include"]: + native_cli = f'hydraJobs.{row["system"]}.exe:simplex-chat' + with self.subTest(row=row): + self.assertEqual(step_enabled("Check native CLI encryption and profile passwords", row), + native_cli in listed_outputs(row["system"], row["target"])) + + def test_split_artifacts_and_existing_execution_bounds(self): + source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text() + template = "nix-${{ matrix.system }}-${{ matrix.target }}-${{ github.sha }}" + self.assertEqual(source.count("name: " + template), 1) + names = [template.replace("${{ matrix.system }}", row["system"]) + .replace("${{ matrix.target }}", row["target"]) + .replace("${{ github.sha }}", "same-source-sha") + for row in matrix.select("all")["include"]] + self.assertEqual(len(set(names)), len(names)) + self.assertRegex(source, r"(?m)^ timeout-minutes: 360$") + self.assertRegex(source, r"(?m)^ NIX_BUILD_TIMEOUT: 7200$") + self.assertRegex(source, r"(?m)^ fail-fast: false$") + self.assertIn("timeout-minutes: 1", workflow_step("Check Intel Darwin build policies")) + self.assertIn("timeout-minutes: 10", workflow_step("Check native Intel Darwin GHC info-table placement")) + self.assertIn("timeout-minutes: 120", workflow_step("Capture Intel Darwin ghc-boot Setup failure")) + def test_manual_groups_have_distinct_queues_without_changing_pr_or_push_keys(self): source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text() groups = re.findall(r"^ group: (.*)$", source, re.MULTILINE) self.assertEqual(len(groups), 1) + self.assertRegex(source, r"(?m)^ cancel-in-progress: false$") prefix = "nix-${{ github.workflow }}-${{ github.ref }}" self.assertTrue(groups[0].startswith(prefix)) suffix = groups[0][len(prefix):] @@ -85,11 +186,11 @@ class Matrix(unittest.TestCase): return "nix-Nix Haskell builds-refs/heads/branch" + suffix_value for event in ["pull_request", "push"]: - for group in [None, "all", "linux-static", "android", "windows", "x86_64-darwin"]: + for group in [None, "all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"]: self.assertEqual(queue_key(event, group), "nix-Nix Haskell builds-refs/heads/branch") keys = [queue_key("workflow_dispatch", group) - for group in ["all", "linux-static", "android", "windows", "x86_64-darwin"]] - self.assertEqual(len(set(keys)), 5) + for group in ["all", "linux-static", "android", "windows", "x86_64-darwin", "x86_64-darwin-libs"]] + self.assertEqual(len(set(keys)), 6) self.assertEqual(queue_key("workflow_dispatch", ""), keys[0]) self.assertEqual(queue_key("workflow_dispatch", None), keys[0])