From 16618c26cca822634fdcf604c92dc7acfa6d4204 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Mon, 19 Oct 2026 22:36:12 +0100 Subject: [PATCH] ci: reduce runners (#7778) * ci: reduce runners * ram disk in mac tests --------- Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com> --- .github/workflows/build.yml | 66 +++++++++++++++++++++++------ .github/workflows/packages.yml | 2 - docs/contributing/RELEASES.md | 6 +-- scripts/ci/packages_release.py | 28 +++++------- scripts/ci/test_packages_release.py | 51 ++++++++++++++++++---- 5 files changed, 108 insertions(+), 45 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68737fdb3f..3748eccbd3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -222,13 +222,6 @@ jobs: fail-fast: false matrix: include: - - os: 22.04 - os_underscore: 22_04 - arch: x86_64 - runner: "ubuntu-22.04" - ghc: "8.10.7" - hash: 'sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5' - should_run: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} - os: 22.04 os_underscore: 22_04 arch: x86_64 @@ -629,7 +622,7 @@ jobs: - name: Setup Node.js id: setup-node - if: ${{ !cancelled() && ((needs.maybe-release.outputs.core_changed && steps.build-lib.outcome == 'success') || (matrix.libs && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish))) }} + if: ${{ !cancelled() && matrix.libs && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} uses: actions/setup-node@v7 with: node-version: 22 @@ -642,11 +635,11 @@ jobs: python-version: "3.12" - name: Package tests (pull request) - if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.core_changed }} + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' && steps.build-lib.outcome != 'failure' }} timeout-minutes: 30 shell: bash env: - SIMPLEX_LIBS_DIR: ${{ github.workspace }}/release-libs/simplex-chat-libs-linux-${{ matrix.arch }}/libs + SIMPLEX_LIBS_DIR: ${{ steps.build-lib.outcome == 'success' && format('{0}/release-libs/simplex-chat-libs-linux-{1}/libs', github.workspace, matrix.arch) || '' }} run: | scripts/ci/package-tests.sh build linux-${{ matrix.arch }} scripts/ci/package-tests.sh test @@ -757,6 +750,31 @@ jobs: hash: 'sha256:68434214381cb38287104e629fe8ee720167dd98cbb36ab1cbbab342515fa6ab' steps: *linux-steps + build-linux-ghc8: + name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}" + needs: [maybe-release, variables] + if: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }} + runs-on: ${{ matrix.runner }} + permissions: + contents: write + packages: write + env: + image_cache: ghcr.io/simplex-chat/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }} + source_hashes: dist-newstyle/sources.sha256 + concurrency: *linux-concurrency + strategy: + fail-fast: false + matrix: + include: + - os: 22.04 + os_underscore: 22_04 + arch: x86_64 + runner: "ubuntu-22.04" + ghc: "8.10.7" + hash: 'sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5' + should_run: true + steps: *linux-steps + # ================================= # Linux PostgreSQL Library Build # ================================= @@ -1093,6 +1111,14 @@ jobs: github_ref: ${{ github.ref }} github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Mount RAM disk for tests + if: matrix.arch == 'aarch64' + shell: bash + run: | + disk=$(hdiutil attach -nomount ram://1048576 | tr -d '[:space:]') + diskutil erasevolume HFS+ tests "$disk" + ln -s /Volumes/tests tests/tmp + - name: Run tests if: matrix.arch == 'aarch64' timeout-minutes: 120 @@ -1141,7 +1167,7 @@ jobs: - name: Setup Node.js id: setup-node - if: ${{ !cancelled() && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} + if: ${{ !cancelled() && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} uses: actions/setup-node@v7 with: node-version: 22 @@ -1153,6 +1179,14 @@ jobs: with: python-version: "3.12" + - name: Package tests (pull request) + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' }} + timeout-minutes: 30 + shell: bash + run: | + scripts/ci/package-tests.sh build macos-${{ matrix.arch }} + scripts/ci/package-tests.sh test + - name: Add-on if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }} timeout-minutes: 30 @@ -1402,7 +1436,7 @@ jobs: - name: Setup Node.js id: setup-node - if: ${{ !cancelled() && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} + if: ${{ !cancelled() && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }} uses: actions/setup-node@v7 with: node-version: 22 @@ -1414,6 +1448,14 @@ jobs: with: python-version: "3.12" + - name: Package tests (pull request) + if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' }} + timeout-minutes: 30 + shell: bash + run: | + scripts/ci/package-tests.sh build windows-x86_64 + scripts/ci/package-tests.sh test + - name: Add-on if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }} timeout-minutes: 30 diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index e5ed17c4f8..ca22698096 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -26,7 +26,6 @@ jobs: runs-on: ubuntu-latest outputs: platforms: ${{ steps.plan.outputs.platforms }} - test: ${{ steps.plan.outputs.test }} steps: - name: Checkout Code uses: actions/checkout@v6 @@ -70,5 +69,4 @@ jobs: run: scripts/ci/package-tests.sh build ${{ matrix.platform }} - name: Test simplex-chat (npm and PyPI) - if: needs.addon.outputs.test run: scripts/ci/package-tests.sh test diff --git a/docs/contributing/RELEASES.md b/docs/contributing/RELEASES.md index e0b80fda5b..17d54ef76c 100644 --- a/docs/contributing/RELEASES.md +++ b/docs/contributing/RELEASES.md @@ -148,7 +148,5 @@ Versions unpublished after a failure: published by the next push to the branch. ## Pull requests -- `packages.yml`, for package changes: - - core unchanged: add-on built and packages tested on 5 platforms - - core changed: add-on built on 5 platforms when its sources changed -- `build.yml`, for core changes: package tests on Linux x86_64 with the library built from the pull request +- `build.yml`, for every pull request it runs for: add-on built and packages tested on 4 platforms (Linux aarch64 excluded); Linux x86_64 uses the library built from the pull request when the core changed, other platforms the pinned library +- `packages.yml`, for package changes in a pull request without `build.yml` paths: add-on built and packages tested on 5 platforms diff --git a/scripts/ci/packages_release.py b/scripts/ci/packages_release.py index fdfd7c3b68..7f7556e900 100644 --- a/scripts/ci/packages_release.py +++ b/scripts/ci/packages_release.py @@ -11,7 +11,7 @@ check-pins check the pins of the packages against the published releases download download a release asset and check its digest download-pinned download a pinned asset and check its pin changes print whether the changes since a commit touch the chat core and the add-on -build-plan print the platforms to build the add-on on and whether to run the package tests +build-plan print the platforms to test the packages on, none when build.yml runs for the pull request prepare update pins and versions of the packages publish-plan print the package versions that are not published yet record record releases and published versions in releases.toml of simplex-chat-libs @@ -47,6 +47,7 @@ RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts" PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py" ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"] CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"] +BUILD_WORKFLOW = ".github/workflows/build.yml" CABAL_FILE = "simplex-chat.cabal" CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M) TYPES_SOURCES = [f"{TYPES_DIR}/src"] @@ -357,17 +358,11 @@ def plan(state: State) -> Plan: ) -@dataclass(frozen=True) -class BuildPlan: - platforms: list[str] - test: bool - - -def build_plan(core_changed: bool, addon_changed: bool) -> BuildPlan: - """The package tests of a pull request that changes the core run in build.yml, with its own libsimplex.""" - if core_changed: - return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False) - return BuildPlan(ADDON_PLATFORMS, test=True) +def pull_request_paths(workflow: str) -> list[str]: + found = re.search(r'^ pull_request:\n paths:\n((?: - ".*"\n)+)', workflow, re.M) + if not found: + raise ValueError(f"no pull_request paths in {BUILD_WORKFLOW}") + return re.findall(r'^ - "(.*)"$', found[1], re.M) def build_matrix(platforms: list[str]) -> str: @@ -491,12 +486,9 @@ def changes(base: str) -> None: def print_build_plan(args: argparse.Namespace) -> None: - result = build_plan( - core_changed=changed_since(args.base, CORE_SOURCES), - addon_changed=changed_since(args.base, ADDON_SOURCES), - ) - print(f"platforms={build_matrix(result.platforms)}") - print(f"test={'true' if result.test else ''}") + build_paths = [f":(glob){path}" for path in pull_request_paths(read_text(BUILD_WORKFLOW))] + platforms = [] if changed_since(args.base, build_paths) else ADDON_PLATFORMS + print(f"platforms={build_matrix(platforms)}") def package_state( diff --git a/scripts/ci/test_packages_release.py b/scripts/ci/test_packages_release.py index 994e678690..3cbd42689a 100644 --- a/scripts/ci/test_packages_release.py +++ b/scripts/ci/test_packages_release.py @@ -25,7 +25,7 @@ from packages_release import ( TYPES_DIR, TYPES_README, ADDON_PLATFORMS, - BuildPlan, + BUILD_WORKFLOW, LIBS_ASSETS, Package, Pins, @@ -34,7 +34,6 @@ from packages_release import ( addon_notes, addon_release, build_matrix, - build_plan, bump_npm_version, bump_python_version, changed_since, @@ -48,7 +47,9 @@ from packages_release import ( package_state, pin_problems, prepare, + print_build_plan, publish_plan, + pull_request_paths, record_releases, recorded_commit, release_status, @@ -238,13 +239,11 @@ class PlanTest(unittest.TestCase): class BuildPlanTest(unittest.TestCase): - def test_pull_request_tests_with_the_pinned_libs(self): - self.assertEqual(build_plan(core_changed=False, addon_changed=False), BuildPlan(ADDON_PLATFORMS, test=True)) - self.assertEqual(build_plan(core_changed=False, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=True)) - - def test_core_change_leaves_tests_to_build_yml(self): - self.assertEqual(build_plan(core_changed=True, addon_changed=False), BuildPlan([], test=False)) - self.assertEqual(build_plan(core_changed=True, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=False)) + def test_reads_the_pull_request_paths_of_build_yml(self): + paths = pull_request_paths((packages_release.ROOT / BUILD_WORKFLOW).read_text()) + self.assertIn("src/**", paths) + self.assertIn(".github/**", paths) + self.assertFalse([path for path in paths if path.startswith("packages/")]) def test_matrix_names_the_runner_of_each_platform(self): self.assertEqual( @@ -406,6 +405,40 @@ class GitRepoTest(unittest.TestCase): ).stdout.strip() +class PrintBuildPlanTest(GitRepoTest): + def setUp(self): + super().setUp() + self.write(BUILD_WORKFLOW, 'on:\n pull_request:\n paths:\n - "src/**"\n - "Dockerfile*"\n\nenv:\n') + self.write("packages/pkg/src.txt", "a") + self.commit("base") + self.base = self.head() + + def platforms(self): + output = io.StringIO() + with mock.patch.object(packages_release, "ROOT", self.repo), contextlib.redirect_stdout(output): + print_build_plan(argparse.Namespace(base=self.base)) + return [entry["platform"] for entry in json.loads(output.getvalue().removeprefix("platforms="))] + + def test_package_change_is_tested_on_every_platform(self): + self.write("packages/pkg/src.txt", "b") + self.commit("package change") + self.assertEqual(self.platforms(), ADDON_PLATFORMS) + + def test_change_that_runs_build_yml_is_tested_there(self): + self.write("packages/pkg/src.txt", "b") + self.write("src/Lib/Module.hs", "m") + self.commit("package and core change") + self.assertEqual(self.platforms(), []) + + def test_patterns_match_as_path_filters(self): + self.write("packages/pkg/Dockerfile", "d") + self.commit("nested Dockerfile") + self.assertEqual(self.platforms(), ADDON_PLATFORMS) + self.write("Dockerfile.build", "d") + self.commit("top-level Dockerfile") + self.assertEqual(self.platforms(), []) + + class PackageStateTest(GitRepoTest): def setUp(self): super().setUp()