mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-11-11 18:17:46 +00:00
Bound the canonical Windows TH gate and trace failed interpreter calls
This commit is contained in:
5 files changed
+485
No files matched your search
@@ -108,6 +108,22 @@ jobs:
|
||||
timeout-minutes: 120
|
||||
run: nix build .#checks.x86_64-linux.ghc-elf-alignment --no-link -L --max-jobs 1 --timeout "$NIX_BUILD_TIMEOUT"
|
||||
|
||||
# Prepare only the exact TH component's direct inputs with the normal
|
||||
# budget. Its own canonical build then gets five minutes, not two hours.
|
||||
- name: Prepare canonical Windows TH inputs
|
||||
id: windows_th_inputs
|
||||
if: ${{ matrix.system == 'x86_64-linux' && matrix.target == 'windows' }}
|
||||
timeout-minutes: 240
|
||||
shell: bash
|
||||
run: python3 scripts/nix/windows-th-gate.py prepare 2>&1 | tee windows-th-inputs.log
|
||||
|
||||
- name: Check canonical Windows TH component
|
||||
id: windows_th_gate
|
||||
if: ${{ matrix.system == 'x86_64-linux' && matrix.target == 'windows' }}
|
||||
timeout-minutes: 6
|
||||
shell: bash
|
||||
run: python3 scripts/nix/windows-th-gate.py build 2>&1 | tee windows-th-gate.log
|
||||
|
||||
# Use the compiler fork in flake.lock, including both Android ABIs.
|
||||
# Stock GHC installed by ghcup is not interchangeable with this compiler.
|
||||
- name: Build existing Haskell outputs
|
||||
@@ -120,6 +136,27 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
run: nix build .#checks.x86_64-darwin.ghc-macho-info-tables --no-link -L --max-jobs 1 --timeout 300
|
||||
|
||||
- name: Trace failed canonical Windows TH component
|
||||
id: windows_th_trace
|
||||
if: ${{ failure() && matrix.system == 'x86_64-linux' && matrix.target == 'windows' && steps.windows_th_inputs.outcome == 'success' && steps.windows_th_gate.outcome == 'failure' && steps.windows_th_gate.outputs.attempted == 'true' }}
|
||||
timeout-minutes: 15
|
||||
shell: bash
|
||||
run: |
|
||||
timeout --kill-after=30s 840s nice -n 10 \
|
||||
python3 scripts/nix/windows-th-gate.py trace 2>&1 | tee windows-th-trace.log
|
||||
|
||||
- name: Upload Windows TH gate and diagnostic
|
||||
if: ${{ always() && matrix.system == 'x86_64-linux' && matrix.target == 'windows' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: nix-windows-th-${{ github.sha }}
|
||||
path: |
|
||||
windows-th-gate.json
|
||||
windows-th-inputs.log
|
||||
windows-th-gate.log
|
||||
windows-th-trace.log
|
||||
if-no-files-found: warn
|
||||
|
||||
# Diagnose the same native library after failure. Keep the canonical
|
||||
# failure and its flags intact; this is a separate diagnostic derivation.
|
||||
- name: Trace native Linux GHC and capture faults after failure
|
||||
|
||||
@@ -195,6 +195,23 @@ host, target group, and source SHA. Each native CLI also runs
|
||||
the encrypted-store and profile-password smoke test, which can be repeated with
|
||||
`python3 scripts/nix/smoke-cli.py result-nix/BUILD_SYSTEM/exe:simplex-chat/bin/simplex-chat`.
|
||||
|
||||
The Windows job first prepares only the named direct input outputs of the
|
||||
canonical `wai-app-static` component from the same Windows CLI project, using
|
||||
the normal 7,200-second per-derivation limit. It then builds that exact component
|
||||
with a 300-second limit; success is reused by the full Windows build. A failure
|
||||
keeps the job red and skips the full outputs. CI replays only that failed
|
||||
component in a separate diagnostic derivation, with `ISERV_ARGS=-v` and
|
||||
`strace -f -tt -T -s 80` around compiler calls. Compiler queries, original GHC
|
||||
arguments, the Wine/DLL wrapper and its normal/profile selection stay unchanged.
|
||||
The trace has a 15-minute step bound and a 300-second per-derivation limit;
|
||||
strace is fetched from caches only. Logs and the selected derivation/input
|
||||
metadata are uploaded even on failure. Repeat the canonical gate on Linux with
|
||||
`python3 scripts/nix/windows-th-gate.py prepare` followed by
|
||||
`python3 scripts/nix/windows-th-gate.py build`; use `trace` only after a failed
|
||||
canonical attempt. These commands use the committed checkout and preserve the
|
||||
canonical Nix core setting. The diagnostic changes its derivation and process
|
||||
environment; it does not qualify a Windows artifact or change the failed result.
|
||||
|
||||
After a failed native x86_64 Linux library build, CI tries a separate derivation
|
||||
with `-v2 -ddump-timings` and GDB on compile calls and uploads
|
||||
`native-linux-ghc-phase-trace.log`, including when that trace fails. Compiler
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Focused controls for input selection and the compile-only Windows trace."""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
spec = importlib.util.spec_from_file_location("gate", HERE / "windows-th-gate.py")
|
||||
gate = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(gate)
|
||||
SELECTED = "/nix/store/" + "0" * 32 + "-" + gate.COMPONENT + ".drv"
|
||||
DEP = "/nix/store/" + "1" * 32 + "-dependency.drv"
|
||||
NEEDLE = ' printf \'#!/bin/sh\\nexec %s "$@"\\n\' "$g" > "$wrapBin/$gName"'
|
||||
|
||||
|
||||
def document(modern=True):
|
||||
row = {"system": "x86_64-linux", "env": {"name": gate.COMPONENT},
|
||||
"outputs": {"out": {"path": "unused"}}}
|
||||
if modern:
|
||||
row.update(version=4, inputs={"srcs": [], "drvs": {
|
||||
Path(DEP).name: {"outputs": ["out", "dev"], "dynamicOutputs": {}}}})
|
||||
return {"version": 4, "derivations": {Path(SELECTED).name: row}}
|
||||
row["inputDrvs"] = {DEP: ["out", "dev"]}
|
||||
return {SELECTED: row}
|
||||
|
||||
|
||||
class Inputs(unittest.TestCase):
|
||||
def test_both_schemas_preserve_requested_outputs(self):
|
||||
for modern in [True, False]:
|
||||
with self.subTest(modern=modern):
|
||||
self.assertEqual(gate.direct_inputs(document(modern), SELECTED), [DEP + "^out,dev"])
|
||||
|
||||
def test_rejects_unknown_dynamic_unsafe_or_wrong_component(self):
|
||||
cases = []
|
||||
def changed(edit):
|
||||
value = document()
|
||||
edit(value, next(iter(value["derivations"].values())))
|
||||
cases.append(value)
|
||||
changed(lambda d, r: d.update(version=5))
|
||||
changed(lambda d, r: r.update(version=3))
|
||||
changed(lambda d, r: r.update(system="aarch64-linux"))
|
||||
changed(lambda d, r: r["env"].update(name="another-component"))
|
||||
changed(lambda d, r: r["outputs"].update(dev={}))
|
||||
changed(lambda d, r: r["inputs"].update(future={}))
|
||||
changed(lambda d, r: r["inputs"]["drvs"][Path(DEP).name].update(dynamicOutputs={"out": {}}))
|
||||
changed(lambda d, r: r["inputs"]["drvs"][Path(DEP).name].update(future={}))
|
||||
for outputs in [[], ["out", "out"], ["*"], ["out,dev"], ["../bad"], [1]]:
|
||||
changed(lambda d, r, outputs=outputs: r["inputs"]["drvs"][Path(DEP).name].update(outputs=outputs))
|
||||
for key in ["../dependency.drv", "/tmp/" + Path(DEP).name, "bad.drv"]:
|
||||
changed(lambda d, r, key=key: r["inputs"].update(drvs={key: {"outputs": ["out"], "dynamicOutputs": {}}}))
|
||||
changed(lambda d, r: r["inputs"]["drvs"].update({DEP: {"outputs": ["out"], "dynamicOutputs": {}}}))
|
||||
changed(lambda d, r: r["inputs"]["drvs"].update({Path(SELECTED).name: {"outputs": ["out"], "dynamicOutputs": {}}}))
|
||||
cases += [{}, [], {"version": 4}, {"version": 4, "derivations": {}, "other": 1}]
|
||||
for index, value in enumerate(cases):
|
||||
with self.subTest(case=index), self.assertRaises(ValueError):
|
||||
gate.direct_inputs(value, SELECTED)
|
||||
legacy = document(False)
|
||||
row = legacy.pop(SELECTED)
|
||||
legacy[Path(SELECTED).name] = row
|
||||
with self.assertRaises(ValueError):
|
||||
gate.direct_inputs(legacy, SELECTED)
|
||||
|
||||
def test_empty_input_set_never_builds_default_flake(self):
|
||||
value = document()
|
||||
next(iter(value["derivations"].values()))["inputs"]["drvs"] = {}
|
||||
with tempfile.TemporaryDirectory() as temp, patch.object(gate, "flake_context", return_value=("a" * 40, "ref", "original")), patch.object(gate, "call", side_effect=[SELECTED, json.dumps(value)]) as call:
|
||||
state = Path(temp) / "state.json"
|
||||
gate.prepare(state)
|
||||
self.assertEqual(call.call_count, 2)
|
||||
initial = call.call_args_list[0]
|
||||
self.assertEqual(initial.args[0][-4:], ["--max-jobs", "1", "--timeout", "7200"])
|
||||
self.assertGreater(initial.kwargs["timeout"], 14000)
|
||||
self.assertTrue(json.loads(state.read_text())["inputsReady"])
|
||||
|
||||
def test_canonical_failure_is_saved_and_trace_requires_it(self):
|
||||
state = dict(revision="a" * 40, flakeRef="ref", original="original", drv=SELECTED,
|
||||
derivation=document(), inputGoals=[DEP + "^out,dev"], inputsReady=True)
|
||||
with tempfile.TemporaryDirectory() as temp, patch.object(gate, "flake_context", return_value=("a" * 40, "ref", "original")):
|
||||
path = Path(temp) / "state.json"
|
||||
path.write_text(json.dumps(state))
|
||||
with self.assertRaises(ValueError):
|
||||
gate.trace(path)
|
||||
with patch.object(gate, "call", side_effect=subprocess.CalledProcessError(1, "nix")) as call:
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
gate.build(path)
|
||||
argv = call.call_args.args[0]
|
||||
self.assertIn(SELECTED + "^out", argv)
|
||||
self.assertNotIn("--cores", argv)
|
||||
self.assertEqual(argv[-2:], ["--timeout", "300"])
|
||||
self.assertEqual(json.loads(path.read_text())["canonicalResult"], "failure")
|
||||
with patch.object(gate, "call") as call:
|
||||
gate.trace(path)
|
||||
tool, replay = [c.args[0] for c in call.call_args_list]
|
||||
self.assertIn("--max-jobs", tool)
|
||||
self.assertEqual(tool[tool.index("--max-jobs") + 1], "0")
|
||||
self.assertEqual(tool[tool.index("--builders") + 1], "")
|
||||
self.assertIn("[]", tool)
|
||||
self.assertIn(SELECTED, replay)
|
||||
self.assertNotIn("--cores", replay)
|
||||
with patch.object(gate, "flake_context", return_value=("b" * 40, "ref", "original")), self.assertRaises(ValueError):
|
||||
gate.read_state(path)
|
||||
|
||||
|
||||
class Wrapper(unittest.TestCase):
|
||||
def render(self, phase, temp, **overrides):
|
||||
path = Path(temp) / "phase.sh"
|
||||
path.write_text(phase)
|
||||
original = dict(system="x86_64-linux", name=gate.COMPONENT,
|
||||
identifier={"name": "wai-app-static"}, drvPath=SELECTED)
|
||||
original.update(overrides)
|
||||
# Pure fake derivation uses the actual override callback and builder seam.
|
||||
attrs = " ".join(f"{k} = builtins.fromJSON {json.dumps(json.dumps(v))};" for k, v in original.items())
|
||||
expression = f'''let original = {{ {attrs}
|
||||
overrideAttrs = f: f {{ buildPhase = builtins.readFile {path}; nativeBuildInputs = [ "compiler" ]; }};
|
||||
}}; in import {HERE / 'trace-windows-th.nix'} {{ inherit original;
|
||||
strace = {json.dumps(str(Path(temp) / 'trace'))}; expectedDrv = {json.dumps(SELECTED)}; }}'''
|
||||
return subprocess.run(["nix", "eval", "--impure", "--json", "--expr", expression], capture_output=True, text=True)
|
||||
|
||||
def test_unique_seam_and_component_guards(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
for phase in ["no seam", NEEDLE + "\n" + NEEDLE]:
|
||||
with self.subTest(phase=phase):
|
||||
self.assertNotEqual(self.render(phase, temp).returncode, 0)
|
||||
for changes in [{"system": "aarch64-linux"}, {"name": "other"}, {"identifier": {"name": "other"}}, {"drvPath": DEP}]:
|
||||
with self.subTest(changes=changes):
|
||||
self.assertNotEqual(self.render(NEEDLE, temp, **changes).returncode, 0)
|
||||
|
||||
def test_actual_rendered_shell_preserves_arguments_and_status(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
temp = Path(temp)
|
||||
(temp / "trace/bin").mkdir(parents=True)
|
||||
(temp / "wrappers").mkdir()
|
||||
fake = '''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
kind = os.path.basename(sys.argv[0])
|
||||
with open(os.environ["CAPTURE"], "a") as f:
|
||||
f.write(json.dumps({"kind": kind, "args": sys.argv[1:], "iserv": os.environ.get("ISERV_ARGS")}) + "\\n")
|
||||
if kind == "strace": os.execv(sys.argv[6], sys.argv[6:])
|
||||
if kind == "timeout": os.execv(sys.argv[3], sys.argv[3:])
|
||||
print("query-control")
|
||||
sys.exit(int(os.environ.get("GHC_EXIT", "0")))
|
||||
'''
|
||||
for name in ["ghc", "timeout", "trace/bin/strace"]:
|
||||
path = temp / name
|
||||
path.write_text(fake)
|
||||
path.chmod(0o755)
|
||||
phase = f'g={temp}/ghc\nwrapBin={temp}/wrappers\ngName=ghc\n' + NEEDLE + '\nchmod +x "$wrapBin/$gName"\nexit 0\n'
|
||||
rendered = self.render(phase, temp)
|
||||
self.assertEqual(rendered.returncode, 0, rendered.stderr)
|
||||
result = json.loads(rendered.stdout)
|
||||
self.assertEqual(result["nativeBuildInputs"], ["compiler", str(temp / "trace")])
|
||||
phasepath = temp / "rendered.sh"
|
||||
phasepath.write_text(result["buildPhase"])
|
||||
subprocess.run(["bash", "-n", str(phasepath)], check=True)
|
||||
subprocess.run(["bash", str(phasepath)], check=True)
|
||||
wrapper = temp / "wrappers/ghc"
|
||||
subprocess.run(["sh", "-n", str(wrapper)], check=True)
|
||||
for args, compile_call in [(["--numeric-version"], False), (["@some response", "--info"], False), (["-M", "-c"], False), (["--make", "a b.hs", "-j4", "-j1"], True), (["-c", "quote'and\"space", "-pgmi", "/original/wine wrapper", "-opti", "unchanged"], True), (["@some response"], True), (["unrecognized"], False)]:
|
||||
for status in [0, 7, 139]:
|
||||
capture = temp / "capture.jsonl"
|
||||
capture.write_text("")
|
||||
env = dict(os.environ, CAPTURE=str(capture), GHC_EXIT=str(status),
|
||||
ISERV_ARGS="ambient", NIX_BUILD_TOP=str(temp),
|
||||
PATH=str(temp) + os.pathsep + os.environ["PATH"])
|
||||
actual = subprocess.run([str(wrapper), *args], env=env, capture_output=True, text=True)
|
||||
self.assertEqual(actual.returncode, status)
|
||||
records = [json.loads(line) for line in capture.read_text().splitlines()]
|
||||
self.assertEqual([r["kind"] for r in records], ["timeout", "strace", "ghc"] if compile_call else ["ghc"])
|
||||
self.assertEqual(records[-1]["args"], args)
|
||||
self.assertEqual(records[-1]["iserv"], "-v" if compile_call else "ambient")
|
||||
self.assertEqual(actual.stdout, "query-control\n")
|
||||
if compile_call:
|
||||
self.assertEqual(records[1]["args"][:5], ["-f", "-tt", "-T", "-s", "80"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,48 @@
|
||||
# Failure-only replay of the exact canonical Windows TH component. Source,
|
||||
# plan, compiler arguments, Wine wrapper and normal/profile selection stay put.
|
||||
{ original, strace, expectedDrv }:
|
||||
let
|
||||
needle = " printf '#!/bin/sh\\nexec %s \"$@\"\\n' \"$g\" > \"$wrapBin/$gName\"";
|
||||
replacement = ''
|
||||
printf '#!/bin/sh\nrealGhc="%s"\n' "$g" > "$wrapBin/$gName"
|
||||
cat >> "$wrapBin/$gName" <<'SIMPLEX_WINDOWS_TH_TRACE'
|
||||
ghcTraceCompile=
|
||||
for arg do
|
||||
case "$arg" in
|
||||
--numeric-version|--version|-V|--info|--print-*|--show-*|--supported-*|--abi-hash|-M)
|
||||
exec "$realGhc" "$@"
|
||||
;;
|
||||
@*|--make|-c) ghcTraceCompile=1 ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$ghcTraceCompile" ]; then
|
||||
# The pinned Wine wrapper appends this to the child command, where both
|
||||
# the native proxy and target interpreter recognize -v. No -opti or
|
||||
# PROXY_ARGS change: their positions differ from this protocol's --pipe.
|
||||
export ISERV_ARGS=-v
|
||||
: "''${NIX_BUILD_TOP:?Windows TH diagnostic requires NIX_BUILD_TOP}"
|
||||
touch "$NIX_BUILD_TOP/windows-th-trace-attempted"
|
||||
exec timeout --kill-after=30s 300s ${strace}/bin/strace -f -tt -T -s 80 "$realGhc" "$@"
|
||||
fi
|
||||
exec "$realGhc" "$@"
|
||||
SIMPLEX_WINDOWS_TH_TRACE
|
||||
'';
|
||||
occurrences = phase: builtins.div
|
||||
(builtins.stringLength phase - builtins.stringLength
|
||||
(builtins.replaceStrings [ needle ] [ "" ] phase))
|
||||
(builtins.stringLength needle);
|
||||
in
|
||||
assert original.system == "x86_64-linux";
|
||||
assert original.identifier.name == "wai-app-static";
|
||||
assert original.name == "wai-app-static-lib-wai-app-static-x86_64-w64-mingw32-3.1.9.1";
|
||||
assert original.drvPath == expectedDrv;
|
||||
original.overrideAttrs (old:
|
||||
assert occurrences old.buildPhase == 1; {
|
||||
nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ strace ];
|
||||
buildPhase = builtins.replaceStrings [ needle ] [ replacement ] old.buildPhase + ''
|
||||
if [ ! -f "$NIX_BUILD_TOP/windows-th-trace-attempted" ]; then
|
||||
echo 'ERROR: Windows TH diagnostic did not intercept a compile call' >&2
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
})
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the canonical Windows TH component before the full Windows outputs.
|
||||
|
||||
Only its direct derivation inputs are prepared under the normal build budget.
|
||||
The component then has a short budget; its failure remains a CI failure even
|
||||
when the separate strace diagnostic succeeds.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
STORE = "/nix/store/"
|
||||
DRV_NAME = re.compile(r"[0-9abcdfghijklmnpqrsvwxyz]{32}-[A-Za-z0-9+._?=-]+\.drv\Z")
|
||||
OUTPUT = re.compile(r"[A-Za-z_][A-Za-z0-9_+-]*\Z")
|
||||
COMPONENT = "wai-app-static-lib-wai-app-static-x86_64-w64-mingw32-3.1.9.1"
|
||||
|
||||
|
||||
def drv_path(value, *, basename=False):
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("derivation path must be a string")
|
||||
name = value[len(STORE):] if value.startswith(STORE) else value if basename else ""
|
||||
if not DRV_NAME.fullmatch(name):
|
||||
raise ValueError(f"invalid derivation path: {value!r}")
|
||||
return STORE + name
|
||||
|
||||
|
||||
def direct_inputs(document, expected):
|
||||
"""Return explicit drv^output goals from supported Nix JSON formats."""
|
||||
expected = drv_path(expected)
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("derivation JSON must be an object")
|
||||
if "derivations" in document or "version" in document:
|
||||
if (set(document) != {"derivations", "version"} or
|
||||
type(document["version"]) is not int or document["version"] != 4):
|
||||
raise ValueError("unsupported derivation JSON version")
|
||||
rows = document["derivations"]
|
||||
modern = True
|
||||
else:
|
||||
rows = document
|
||||
modern = False
|
||||
if not isinstance(rows, dict) or len(rows) != 1:
|
||||
raise ValueError("expected exactly the selected component derivation")
|
||||
key, row = next(iter(rows.items()))
|
||||
if drv_path(key, basename=modern) != expected or not isinstance(row, dict):
|
||||
raise ValueError("selected component derivation does not match")
|
||||
if (row.get("system") != "x86_64-linux" or not isinstance(row.get("env"), dict) or
|
||||
row["env"].get("name") != COMPONENT):
|
||||
raise ValueError("expected the native-Linux-built Windows wai-app-static component")
|
||||
if not isinstance(row.get("outputs"), dict) or set(row["outputs"]) != {"out"}:
|
||||
raise ValueError("expected the component's single out output")
|
||||
if modern:
|
||||
if (type(row.get("version")) is not int or row["version"] != 4 or
|
||||
not isinstance(row.get("inputs"), dict) or set(row["inputs"]) != {"drvs", "srcs"} or
|
||||
not isinstance(row["inputs"]["srcs"], list)):
|
||||
raise ValueError("unsupported derivation input schema")
|
||||
inputs = row["inputs"]["drvs"]
|
||||
else:
|
||||
if "inputs" in row or "version" in row or not isinstance(row.get("inputDrvs"), dict):
|
||||
raise ValueError("unsupported legacy derivation input schema")
|
||||
inputs = row["inputDrvs"]
|
||||
if not isinstance(inputs, dict):
|
||||
raise ValueError("derivation inputs must be an object")
|
||||
goals = []
|
||||
seen = set()
|
||||
for key, value in sorted(inputs.items()):
|
||||
path = drv_path(key, basename=modern)
|
||||
if path == expected or path in seen:
|
||||
raise ValueError("duplicate or self-referencing input derivation")
|
||||
seen.add(path)
|
||||
if modern:
|
||||
if not isinstance(value, dict) or set(value) != {"outputs", "dynamicOutputs"}:
|
||||
raise ValueError("unsupported input derivation schema")
|
||||
if value["dynamicOutputs"] != {}:
|
||||
raise ValueError("dynamic derivation outputs are not supported")
|
||||
outputs = value["outputs"]
|
||||
else:
|
||||
outputs = value
|
||||
if (not isinstance(outputs, list) or not outputs or
|
||||
any(not isinstance(o, str) or not OUTPUT.fullmatch(o) for o in outputs) or
|
||||
len(set(outputs)) != len(outputs)):
|
||||
raise ValueError("expected explicit, unique named input outputs")
|
||||
goals.append(path + "^" + ",".join(outputs))
|
||||
return goals
|
||||
|
||||
|
||||
def call(argv, *, capture=False, timeout=600):
|
||||
print("+ " + " ".join(argv), flush=True)
|
||||
result = subprocess.run(argv, text=True, stdout=subprocess.PIPE if capture else None,
|
||||
check=True, timeout=timeout)
|
||||
return result.stdout if capture else None
|
||||
|
||||
|
||||
def flake_context():
|
||||
root = Path(call(["git", "rev-parse", "--show-toplevel"], capture=True).strip())
|
||||
revision = call(["git", "rev-parse", "HEAD"], capture=True).strip()
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", revision):
|
||||
raise ValueError("expected a committed checkout")
|
||||
ref = root.as_uri().replace("file:", "git+file:", 1) + "?rev=" + revision
|
||||
original = (f'(builtins.getFlake {json.dumps(ref)}).hydraJobs.x86_64-linux.'
|
||||
'"x86_64-windows:exe:simplex-chat".project.hsPkgs.wai-app-static.components.library')
|
||||
return revision, ref, original
|
||||
|
||||
|
||||
def read_state(path):
|
||||
state = json.loads(path.read_text())
|
||||
revision, ref, original = flake_context()
|
||||
if (state["revision"], state["flakeRef"], state["original"]) != (revision, ref, original):
|
||||
raise ValueError("gate state belongs to a different checkout")
|
||||
drv_path(state["drv"])
|
||||
if direct_inputs(state["derivation"], state["drv"]) != state["inputGoals"]:
|
||||
raise ValueError("gate input goals do not match the saved derivation")
|
||||
return state
|
||||
|
||||
|
||||
def prepare(path):
|
||||
# Evaluation can realize planning/compiler IFDs. Give those the same normal
|
||||
# budget as the direct inputs; only the ready component gets 300 seconds.
|
||||
deadline = time.monotonic() + 14400
|
||||
|
||||
def remaining():
|
||||
seconds = deadline - time.monotonic()
|
||||
if seconds <= 0:
|
||||
raise subprocess.TimeoutExpired("Windows TH input preparation", 14400)
|
||||
return seconds
|
||||
|
||||
revision, ref, original = flake_context()
|
||||
drv = call(["nix", "eval", "--impure", "--raw", "--expr", f"({original}).drvPath",
|
||||
"-L", "--keep-failed", "--max-jobs", "1", "--timeout", "7200"],
|
||||
capture=True, timeout=remaining()).strip()
|
||||
drv_path(drv)
|
||||
document = json.loads(call(["nix", "derivation", "show", drv], capture=True,
|
||||
timeout=min(600, remaining())))
|
||||
goals = direct_inputs(document, drv)
|
||||
state = dict(revision=revision, flakeRef=ref, original=original, drv=drv,
|
||||
derivation=document, inputGoals=goals, inputsReady=False)
|
||||
path.write_text(json.dumps(state, indent=2) + "\n")
|
||||
# No bare derivations or ^*: preserve only outputs actually requested by
|
||||
# this canonical component. Empty goals must never build the default flake.
|
||||
if goals:
|
||||
call(["nix", "build", *goals, "--no-link", "-L", "--keep-failed",
|
||||
"--max-jobs", "1", "--timeout", "7200"], timeout=remaining())
|
||||
state["inputsReady"] = True
|
||||
path.write_text(json.dumps(state, indent=2) + "\n")
|
||||
|
||||
|
||||
def build(path):
|
||||
state = read_state(path)
|
||||
if state.get("inputsReady") is not True:
|
||||
raise ValueError("canonical component inputs are not ready")
|
||||
if output := os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(output, "a") as handle:
|
||||
handle.write("attempted=true\n")
|
||||
state["canonicalResult"] = "running"
|
||||
path.write_text(json.dumps(state, indent=2) + "\n")
|
||||
try:
|
||||
call(["nix", "build", state["drv"] + "^out", "--no-link", "-L", "--keep-failed",
|
||||
"--max-jobs", "1", "--timeout", "300"], timeout=330)
|
||||
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
||||
state["canonicalResult"] = "failure"
|
||||
path.write_text(json.dumps(state, indent=2) + "\n")
|
||||
raise
|
||||
state["canonicalResult"] = "success"
|
||||
path.write_text(json.dumps(state, indent=2) + "\n")
|
||||
|
||||
|
||||
def trace(path):
|
||||
state = read_state(path)
|
||||
if state.get("inputsReady") is not True:
|
||||
raise ValueError("canonical component inputs are not ready")
|
||||
if state.get("canonicalResult") != "failure":
|
||||
raise ValueError("diagnostic requires a failed canonical component attempt")
|
||||
tool = f'(import ((builtins.getFlake {json.dumps(state["flakeRef"])}).inputs.nixpkgs) {{ system = "x86_64-linux"; }}).strace'
|
||||
# A cache miss must not turn the diagnostic into a toolchain build.
|
||||
call(["nix", "build", "--impure", "--expr", tool, "--no-link", "--max-jobs", "0",
|
||||
"--builders", "", "--option", "external-builders", "[]"], timeout=120)
|
||||
call(["nix", "build", "--impure", "--file", "scripts/nix/trace-windows-th.nix",
|
||||
"--arg", "original", state["original"], "--arg", "strace", tool,
|
||||
"--argstr", "expectedDrv", state["drv"], "--no-link", "-L", "--keep-failed",
|
||||
"--max-jobs", "1", "--timeout", "300"], timeout=600)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=["prepare", "build", "trace"])
|
||||
parser.add_argument("--state", type=Path, default=Path("windows-th-gate.json"))
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
{"prepare": prepare, "build": build, "trace": trace}[args.action](args.state)
|
||||
except (ValueError, KeyError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
|
||||
print(f"Windows TH {args.action} failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user