diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 7e662b3e74..cdfad58c62 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -8,7 +8,7 @@ on: type: choice required: true default: all - options: [all, linux-static, android] + options: [all, linux-static, android, windows] push: branches: [master, stable, build/stable-ghc-9.14] paths: diff --git a/scripts/nix/README.md b/scripts/nix/README.md index 60a8a21357..972ebec08a 100644 --- a/scripts/nix/README.md +++ b/scripts/nix/README.md @@ -219,7 +219,8 @@ native Linux build therefore leaves the cross-build jobs running. PRs, pushes, and default manual runs retain that complete matrix. A manual dispatch can select the existing three `linux-static` outputs with `gh workflow run nix.yml --ref BRANCH -f group=linux-static`, or the four Android -outputs with `gh workflow run nix.yml --ref BRANCH -f group=android`. The registered +outputs with `gh workflow run nix.yml --ref BRANCH -f group=android`, or the two +Windows outputs with `gh workflow run nix.yml --ref BRANCH -f group=windows`. The registered workflow can run on a branch containing this workflow file; it does not require that file on the default branch. Runs for the same workflow/ref queue without canceling an active run; manual groups have separate queues. Legacy PR build jobs also preserve running @@ -237,15 +238,20 @@ the normal 7,200-second per-derivation limit. It then builds that exact componen with a 300-second limit; success is reused by the full Windows build. A failure keeps the job red and skips the full outputs. CI replays only that failed component in a separate diagnostic derivation, with `ISERV_ARGS=-v` and -`strace -f -tt -T -s 80` around compiler calls. Compiler queries, original GHC -arguments, the Wine/DLL wrapper and its normal/profile selection stay unchanged. +`strace -f -tt -T -s 80` around compiler calls. It copies the exact selected Wine +scripts and appends `+seh,+loaddll` to their pinned `WINEDEBUG` value, with an +assertion that no other script bytes change. Only compile calls receive the +copied `-pgmi` path. Compiler queries, source/plan, original arguments, DLL +collection, Wine prefix and normal/profile selection stay unchanged. The trace has a 15-minute step bound and a 300-second per-derivation limit; strace is fetched from caches only. Logs and the selected derivation/input metadata are uploaded even on failure. Repeat the canonical gate on Linux with `python3 scripts/nix/windows-th-gate.py prepare` followed by `python3 scripts/nix/windows-th-gate.py build`; use `trace` only after a failed canonical attempt. These commands use the committed checkout and preserve the -canonical Nix core setting. The diagnostic changes its derivation and process +canonical Nix core setting. A fresh Wine prefix and full syscall tracing can +consume most of the diagnostic budget; a timeout does not identify a deadlock. +The diagnostic changes its derivation and process environment; it does not qualify a Windows artifact or change the failed result. After a failed native x86_64 Linux library build, CI tries a separate derivation diff --git a/scripts/nix/ci-matrix.py b/scripts/nix/ci-matrix.py index 79566febf4..f69bad8027 100644 --- a/scripts/nix/ci-matrix.py +++ b/scripts/nix/ci-matrix.py @@ -15,7 +15,7 @@ ROWS = ( def select(group): - if group not in ("all", "linux-static", "android"): + if group not in ("all", "linux-static", "android", "windows"): raise ValueError(f"unknown Nix CI group: {group}") rows = ROWS if group == "all" else tuple(row for row in ROWS if row[2] == group) return {"include": [dict(system=system, runner=runner, target=target) @@ -24,5 +24,5 @@ def select(group): if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--group", choices=["all", "linux-static", "android"], default="all") + parser.add_argument("--group", choices=["all", "linux-static", "android", "windows"], default="all") print(json.dumps(select(parser.parse_args().group), separators=(",", ":"))) diff --git a/scripts/nix/test-ci-matrix.py b/scripts/nix/test-ci-matrix.py index 21b1392958..303a6cc85a 100644 --- a/scripts/nix/test-ci-matrix.py +++ b/scripts/nix/test-ci-matrix.py @@ -44,6 +44,12 @@ class Matrix(unittest.TestCase): "hydraJobs.x86_64-linux.armv7a-android:lib:support", ]) + def test_windows_dispatch_is_exactly_the_existing_two_outputs(self): + self.assertEqual(outputs("windows"), [ + "hydraJobs.x86_64-linux.x86_64-windows:exe:simplex-chat", + "hydraJobs.x86_64-linux.x86_64-windows:lib:simplex-chat", + ]) + def test_manual_groups_have_distinct_queues_without_changing_pr_or_push_keys(self): source = (HERE.parent.parent / ".github/workflows/nix.yml").read_text() groups = re.findall(r"^ group: (.*)$", source, re.MULTILINE) @@ -64,10 +70,10 @@ class Matrix(unittest.TestCase): return "nix-Nix Haskell builds-refs/heads/branch" + suffix_value for event in ["pull_request", "push"]: - for group in [None, "all", "linux-static", "android"]: + for group in [None, "all", "linux-static", "android", "windows"]: self.assertEqual(queue_key(event, group), "nix-Nix Haskell builds-refs/heads/branch") - keys = [queue_key("workflow_dispatch", group) for group in ["all", "linux-static", "android"]] - self.assertEqual(len(set(keys)), 3) + keys = [queue_key("workflow_dispatch", group) for group in ["all", "linux-static", "android", "windows"]] + self.assertEqual(len(set(keys)), 4) self.assertEqual(queue_key("workflow_dispatch", ""), keys[0]) self.assertEqual(queue_key("workflow_dispatch", None), keys[0]) @@ -99,8 +105,8 @@ class Matrix(unittest.TestCase): def test_unknown_group_rejected(self): with self.assertRaises(ValueError): - matrix.select("windows") - result = subprocess.run(["python3", str(HERE / "ci-matrix.py"), "--group", "windows"], + matrix.select("unknown") + result = subprocess.run(["python3", str(HERE / "ci-matrix.py"), "--group", "unknown"], capture_output=True, text=True) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") diff --git a/scripts/nix/test-windows-th-gate.py b/scripts/nix/test-windows-th-gate.py index 2a59fb0753..91b5a13b62 100644 --- a/scripts/nix/test-windows-th-gate.py +++ b/scripts/nix/test-windows-th-gate.py @@ -16,6 +16,18 @@ spec.loader.exec_module(gate) SELECTED = "/nix/store/" + "0" * 32 + "-" + gate.COMPONENT + ".drv" DEP = "/nix/store/" + "1" * 32 + "-dependency.drv" NEEDLE = ' printf \'#!/bin/sh\\nexec %s "$@"\\n\' "$g" > "$wrapBin/$gName"' +WINEDEBUG = "WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag" + + +def store_iserv(temp, *, normal=None, profiled=None): + """Import tiny script inputs, not a derivation or a compiler.""" + source = Path(temp) / "iserv-wrapper" + (source / "bin").mkdir(parents=True) + for suffix, content in [("", normal), ("-prof", profiled)]: + path = source / ("bin/iserv-wrapper" + suffix) + path.write_text(content if content is not None else "#!/bin/sh\n" + WINEDEBUG + " true \"$@\"\n") + path.chmod(0o755) + return subprocess.check_output(["nix-store", "--add", str(source)], text=True).strip() + "/bin/iserv-wrapper" def document(modern=True): @@ -135,6 +147,7 @@ class Wrapper(unittest.TestCase): temp = Path(temp) (temp / "trace/bin").mkdir(parents=True) (temp / "wrappers").mkdir() + selected_iserv = store_iserv(temp) fake = '''#!/usr/bin/env python3 import json, os, sys kind = os.path.basename(sys.argv[0]) @@ -147,7 +160,7 @@ sys.exit(int(os.environ.get("GHC_EXIT", "0"))) ''' for name in ["ghc", "timeout", "trace/bin/strace"]: path = temp / name - path.write_text(fake) + path.write_text(fake.replace("import json, os, sys", f"# -pgmi {selected_iserv}\nimport json, os, sys")) path.chmod(0o755) phase = f'g={temp}/ghc\nwrapBin={temp}/wrappers\ngName=ghc\n' + NEEDLE + '\nchmod +x "$wrapBin/$gName"\nexit 0\n' rendered = self.render(phase, temp) @@ -171,12 +184,78 @@ sys.exit(int(os.environ.get("GHC_EXIT", "0"))) self.assertEqual(actual.returncode, status) records = [json.loads(line) for line in capture.read_text().splitlines()] self.assertEqual([r["kind"] for r in records], ["timeout", "strace", "ghc"] if compile_call else ["ghc"]) - self.assertEqual(records[-1]["args"], args) + ghc_args = records[-1]["args"] + if compile_call: + self.assertEqual(ghc_args[:-2], args) + self.assertEqual(ghc_args[-2], "-pgmi") + selected_debug = Path(ghc_args[-1]) + self.assertEqual(selected_debug.name, "iserv-wrapper") + for suffix in ["", "-prof"]: + copied = Path(str(selected_debug) + suffix).read_text() + original = Path(selected_iserv + suffix).read_text() + self.assertEqual(copied.replace(WINEDEBUG + ",+seh,+loaddll", WINEDEBUG), original) + else: + self.assertEqual(ghc_args, args) self.assertEqual(records[-1]["iserv"], "-v" if compile_call else "ambient") self.assertEqual(actual.stdout, "query-control\n") if compile_call: self.assertEqual(records[1]["args"][:5], ["-f", "-tt", "-T", "-s", "80"]) +class WineDebug(unittest.TestCase): + def run_copy(self, ghc, destination): + return subprocess.run(["sh", str(HERE / "windows-th-wine-debug.sh"), str(ghc), str(destination)], + capture_output=True, text=True) + + def test_copies_only_debug_value_for_both_interpreters(self): + with tempfile.TemporaryDirectory() as temp: + temp = Path(temp) + script = """#!/bin/sh +unset configureFlags +REMOTE_ISERV=unchanged +WINEDLLOVERRIDES=unchanged WINEDEBUG_VALUE WINEPREFIX="$TMP" python3 -c 'import json,os,sys; print(json.dumps([os.environ["WINEDEBUG"],os.environ["WINEPREFIX"],sys.argv[1:]]))' "$@" +""".replace("WINEDEBUG_VALUE", WINEDEBUG) + original = store_iserv(temp, normal=script, profiled=script.replace("unchanged", "profiled")) + ghc = temp / "ghc" + ghc.write_text("#!/bin/sh\n# -pgmi " + original + "\n") + copied = self.run_copy(ghc, temp / "debug") + self.assertEqual(copied.returncode, 0, copied.stderr) + self.assertEqual(copied.stdout.strip(), str(temp / "debug/iserv-wrapper")) + for suffix in ["", "-prof"]: + target = Path(copied.stdout.strip() + suffix) + subprocess.run(["sh", "-n", str(target)], check=True) + self.assertEqual(target.read_text().replace(WINEDEBUG + ",+seh,+loaddll", WINEDEBUG), + Path(original + suffix).read_text()) + env = dict(os.environ, TMP="same-prefix") + result = subprocess.run([str(target), "a b", "quote'and\"", "-v"], env=env, + capture_output=True, text=True, check=True) + debug, prefix, args = json.loads(result.stdout) + self.assertEqual(debug, WINEDEBUG.removeprefix("WINEDEBUG=") + ",+seh,+loaddll") + self.assertEqual(prefix, "same-prefix") + self.assertEqual(args, ["a b", "quote'and\"", "-v"]) + + def test_rejects_changed_or_ambiguous_originals(self): + for case in ["missing-path", "two-paths", "missing-assignment", "two-assignments", "missing-profile"]: + with self.subTest(case=case), tempfile.TemporaryDirectory() as temp: + temp = Path(temp) + script = "#!/bin/sh\n" + WINEDEBUG + " true\n" + if case == "missing-assignment": script = "#!/bin/sh\ntrue\n" + if case == "two-assignments": script += WINEDEBUG + " true\n" + selected = store_iserv(temp, normal=script) + if case == "missing-profile": + # A valid store path without the paired script must fail, + # without changing any existing store object. + source = temp / "unpaired/iserv-wrapper/bin" + source.mkdir(parents=True) + path = source / "iserv-wrapper" + path.write_text(script) + path.chmod(0o755) + selected = subprocess.check_output(["nix-store", "--add", str(source.parent)], text=True).strip() + "/bin/iserv-wrapper" + ghc = temp / "ghc" + ghc.write_text("none" if case == "missing-path" else selected + ("\n" + selected if case == "two-paths" else "")) + result = self.run_copy(ghc, temp / "debug") + self.assertEqual(result.returncode, 1, result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/nix/trace-windows-th.nix b/scripts/nix/trace-windows-th.nix index eb9e38d66b..6369051ac6 100644 --- a/scripts/nix/trace-windows-th.nix +++ b/scripts/nix/trace-windows-th.nix @@ -1,5 +1,6 @@ # Failure-only replay of the exact canonical Windows TH component. Source, -# plan, compiler arguments, Wine wrapper and normal/profile selection stay put. +# plan and normal/profile selection stay put. A copied Wine wrapper changes +# only its debug-channel value; the canonical component stays untouched. { original, strace, expectedDrv }: let needle = " printf '#!/bin/sh\\nexec %s \"$@\"\\n' \"$g\" > \"$wrapBin/$gName\""; @@ -21,8 +22,10 @@ let # PROXY_ARGS change: their positions differ from this protocol's --pipe. export ISERV_ARGS=-v : "''${NIX_BUILD_TOP:?Windows TH diagnostic requires NIX_BUILD_TOP}" + debugDir=$(mktemp -d "$NIX_BUILD_TOP/windows-th-wine-debug.XXXXXX") || exit 1 + debugPgmi=$(sh ${./windows-th-wine-debug.sh} "$realGhc" "$debugDir") || exit 1 touch "$NIX_BUILD_TOP/windows-th-trace-attempted" - exec timeout --kill-after=30s 300s ${strace}/bin/strace -f -tt -T -s 80 "$realGhc" "$@" + exec timeout --kill-after=30s 300s ${strace}/bin/strace -f -tt -T -s 80 "$realGhc" "$@" -pgmi "$debugPgmi" fi exec "$realGhc" "$@" SIMPLEX_WINDOWS_TH_TRACE diff --git a/scripts/nix/windows-th-wine-debug.sh b/scripts/nix/windows-th-wine-debug.sh new file mode 100644 index 0000000000..e942a3ee3d --- /dev/null +++ b/scripts/nix/windows-th-wine-debug.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Copy the selected Wine iserv scripts, changing only diagnostic channels. +set -eu + +real_ghc=$1 +destination=$2 +needle='WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag' +# Wine parses these left to right. Explicit channel flags follow the global +# class exclusions, so exception and module traces remain enabled. +replacement="$needle,+seh,+loaddll" + +paths=$(grep -oE '/nix/store/[0-9abcdfghijklmnpqrsvwxyz]{32}-iserv-wrapper/bin/iserv-wrapper' "$real_ghc" || true) +count=$(printf '%s\n' "$paths" | grep -c . || true) +if [ "$count" -ne 1 ]; then + echo 'ERROR: expected one original Wine iserv path in the selected GHC wrapper' >&2 + exit 1 +fi +original=$paths +mkdir -p "$destination" +for suffix in '' '-prof'; do + source="$original$suffix" + target="$destination/iserv-wrapper$suffix" + if [ ! -x "$source" ]; then + echo "ERROR: missing selected Wine iserv script: $source" >&2 + exit 1 + fi + count=$(grep -oF "$needle" "$source" | wc -l) + if [ "$count" -ne 1 ]; then + echo "ERROR: expected one pinned WINEDEBUG assignment in $source" >&2 + exit 1 + fi + sed "s/$needle/$replacement/g" "$source" > "$target" + # Refuse any transformation beyond the one debug value. + if ! sed "s/$replacement/$needle/g" "$target" | cmp -s "$source" -; then + echo "ERROR: unexpected change to the selected Wine iserv script: $source" >&2 + exit 1 + fi + chmod 700 "$target" +done +printf 'Windows TH diagnostic: %s; WINEDEBUG=%s\n' "$original" "${replacement#WINEDEBUG=}" >&2 +printf '%s\n' "$destination/iserv-wrapper"