ci: reduce runners (#7778)

* ci: reduce runners

* ram disk in mac tests

---------

Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>

.github/workflows/build.yml

@@ -222,13 +222,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: 22.04
-            os_underscore: 22_04
-            arch: x86_64
-            runner: "ubuntu-22.04"
-            ghc: "8.10.7"
-            hash: 'sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5'
-            should_run: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }}
           - os: 22.04
             os_underscore: 22_04
             arch: x86_64
@@ -629,7 +622,7 @@ jobs:
 
       - name: Setup Node.js
         id: setup-node
-        if: ${{ !cancelled() && ((needs.maybe-release.outputs.core_changed && steps.build-lib.outcome == 'success') || (matrix.libs && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish))) }}
+        if: ${{ !cancelled() && matrix.libs && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }}
         uses: actions/setup-node@v7
         with:
           node-version: 22
@@ -642,11 +635,11 @@ jobs:
           python-version: "3.12"
 
       - name: Package tests (pull request)
-        if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.core_changed }}
+        if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' && steps.build-lib.outcome != 'failure' }}
         timeout-minutes: 30
         shell: bash
         env:
-          SIMPLEX_LIBS_DIR: ${{ github.workspace }}/release-libs/simplex-chat-libs-linux-${{ matrix.arch }}/libs
+          SIMPLEX_LIBS_DIR: ${{ steps.build-lib.outcome == 'success' && format('{0}/release-libs/simplex-chat-libs-linux-{1}/libs', github.workspace, matrix.arch) || '' }}
         run: |
           scripts/ci/package-tests.sh build linux-${{ matrix.arch }}
           scripts/ci/package-tests.sh test
@@ -757,6 +750,31 @@ jobs:
             hash: 'sha256:68434214381cb38287104e629fe8ee720167dd98cbb36ab1cbbab342515fa6ab'
     steps: *linux-steps
 
+  build-linux-ghc8:
+    name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }} (CLI,Desktop), GHC: ${{ matrix.ghc }}"
+    needs: [maybe-release, variables]
+    if: ${{ !(github.ref == 'refs/heads/stable' || startsWith(github.ref, 'refs/tags/v')) }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: write
+      packages: write
+    env:
+      image_cache: ghcr.io/simplex-chat/simplex-chat-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}
+      source_hashes: dist-newstyle/sources.sha256
+    concurrency: *linux-concurrency
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: 22.04
+            os_underscore: 22_04
+            arch: x86_64
+            runner: "ubuntu-22.04"
+            ghc: "8.10.7"
+            hash: 'sha256:5c8b2c0a6c745bc177669abfaa716b4bc57d58e2ea3882fb5da67f4d59e3dda5'
+            should_run: true
+    steps: *linux-steps
+
 # =================================
 #   Linux PostgreSQL Library Build
 # =================================
@@ -1093,6 +1111,14 @@ jobs:
           github_ref: ${{ github.ref }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Mount RAM disk for tests
+        if: matrix.arch == 'aarch64'
+        shell: bash
+        run: |
+          disk=$(hdiutil attach -nomount ram://1048576 | tr -d '[:space:]')
+          diskutil erasevolume HFS+ tests "$disk"
+          ln -s /Volumes/tests tests/tmp
+
       - name: Run tests
         if: matrix.arch == 'aarch64'
         timeout-minutes: 120
@@ -1141,7 +1167,7 @@ jobs:
 
       - name: Setup Node.js
         id: setup-node
-        if: ${{ !cancelled() && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }}
+        if: ${{ !cancelled() && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }}
         uses: actions/setup-node@v7
         with:
           node-version: 22
@@ -1153,6 +1179,14 @@ jobs:
         with:
           python-version: "3.12"
 
+      - name: Package tests (pull request)
+        if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' }}
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          scripts/ci/package-tests.sh build macos-${{ matrix.arch }}
+          scripts/ci/package-tests.sh test
+
       - name: Add-on
         if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }}
         timeout-minutes: 30
@@ -1402,7 +1436,7 @@ jobs:
 
       - name: Setup Node.js
         id: setup-node
-        if: ${{ !cancelled() && (needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }}
+        if: ${{ !cancelled() && (github.event_name == 'pull_request' || needs.maybe-release.outputs.addon_draft || needs.maybe-release.outputs.publish) }}
         uses: actions/setup-node@v7
         with:
           node-version: 22
@@ -1414,6 +1448,14 @@ jobs:
         with:
           python-version: "3.12"
 
+      - name: Package tests (pull request)
+        if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && github.event_name == 'pull_request' }}
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          scripts/ci/package-tests.sh build windows-x86_64
+          scripts/ci/package-tests.sh test
+
       - name: Add-on
         if: ${{ !cancelled() && steps.setup-python.outcome == 'success' && needs.maybe-release.outputs.addon_draft }}
         timeout-minutes: 30

.github/workflows/packages.yml

@@ -26,7 +26,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       platforms: ${{ steps.plan.outputs.platforms }}
-      test: ${{ steps.plan.outputs.test }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v6
@@ -70,5 +69,4 @@ jobs:
         run: scripts/ci/package-tests.sh build ${{ matrix.platform }}
 
       - name: Test simplex-chat (npm and PyPI)
-        if: needs.addon.outputs.test
         run: scripts/ci/package-tests.sh test

docs/contributing/RELEASES.md

@@ -148,7 +148,5 @@ Versions unpublished after a failure: published by the next push to the branch.
 
 ## Pull requests
 
-- `packages.yml`, for package changes:
-  - core unchanged: add-on built and packages tested on 5 platforms
-  - core changed: add-on built on 5 platforms when its sources changed
-- `build.yml`, for core changes: package tests on Linux x86_64 with the library built from the pull request
+- `build.yml`, for every pull request it runs for: add-on built and packages tested on 4 platforms (Linux aarch64 excluded); Linux x86_64 uses the library built from the pull request when the core changed, other platforms the pinned library
+- `packages.yml`, for package changes in a pull request without `build.yml` paths: add-on built and packages tested on 5 platforms

scripts/ci/packages_release.py

@@ -11,7 +11,7 @@ check-pins        check the pins of the packages against the published releases
 download          download a release asset and check its digest
 download-pinned   download a pinned asset and check its pin
 changes           print whether the changes since a commit touch the chat core and the add-on
-build-plan        print the platforms to build the add-on on and whether to run the package tests
+build-plan        print the platforms to test the packages on, none when build.yml runs for the pull request
 prepare           update pins and versions of the packages
 publish-plan      print the package versions that are not published yet
 record            record releases and published versions in releases.toml of simplex-chat-libs
@@ -47,6 +47,7 @@ RELEASE_FILE = f"{NODEJS_DIR}/src/release.ts"
 PYTHON_VERSION_FILE = f"{PYTHON_DIR}/src/simplex_chat/_version.py"
 ADDON_SOURCES = [f"{NODEJS_DIR}/binding.gyp", f"{NODEJS_DIR}/cpp"]
 CORE_SOURCES = ["src", "simplex-chat.cabal", "cabal.project"]
+BUILD_WORKFLOW = ".github/workflows/build.yml"
 CABAL_FILE = "simplex-chat.cabal"
 CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M)
 TYPES_SOURCES = [f"{TYPES_DIR}/src"]
@@ -357,17 +358,11 @@ def plan(state: State) -> Plan:
     )
 
 
-@dataclass(frozen=True)
-class BuildPlan:
-    platforms: list[str]
-    test: bool
-
-
-def build_plan(core_changed: bool, addon_changed: bool) -> BuildPlan:
-    """The package tests of a pull request that changes the core run in build.yml, with its own libsimplex."""
-    if core_changed:
-        return BuildPlan(ADDON_PLATFORMS if addon_changed else [], test=False)
-    return BuildPlan(ADDON_PLATFORMS, test=True)
+def pull_request_paths(workflow: str) -> list[str]:
+    found = re.search(r'^  pull_request:\n    paths:\n((?:      - ".*"\n)+)', workflow, re.M)
+    if not found:
+        raise ValueError(f"no pull_request paths in {BUILD_WORKFLOW}")
+    return re.findall(r'^      - "(.*)"$', found[1], re.M)
 
 
 def build_matrix(platforms: list[str]) -> str:
@@ -491,12 +486,9 @@ def changes(base: str) -> None:
 
 
 def print_build_plan(args: argparse.Namespace) -> None:
-    result = build_plan(
-        core_changed=changed_since(args.base, CORE_SOURCES),
-        addon_changed=changed_since(args.base, ADDON_SOURCES),
-    )
-    print(f"platforms={build_matrix(result.platforms)}")
-    print(f"test={'true' if result.test else ''}")
+    build_paths = [f":(glob){path}" for path in pull_request_paths(read_text(BUILD_WORKFLOW))]
+    platforms = [] if changed_since(args.base, build_paths) else ADDON_PLATFORMS
+    print(f"platforms={build_matrix(platforms)}")
 
 
 def package_state(

scripts/ci/test_packages_release.py

@@ -25,7 +25,7 @@ from packages_release import (
     TYPES_DIR,
     TYPES_README,
     ADDON_PLATFORMS,
-    BuildPlan,
+    BUILD_WORKFLOW,
     LIBS_ASSETS,
     Package,
     Pins,
@@ -34,7 +34,6 @@ from packages_release import (
     addon_notes,
     addon_release,
     build_matrix,
-    build_plan,
     bump_npm_version,
     bump_python_version,
     changed_since,
@@ -48,7 +47,9 @@ from packages_release import (
     package_state,
     pin_problems,
     prepare,
+    print_build_plan,
     publish_plan,
+    pull_request_paths,
     record_releases,
     recorded_commit,
     release_status,
@@ -238,13 +239,11 @@ class PlanTest(unittest.TestCase):
 
 
 class BuildPlanTest(unittest.TestCase):
-    def test_pull_request_tests_with_the_pinned_libs(self):
-        self.assertEqual(build_plan(core_changed=False, addon_changed=False), BuildPlan(ADDON_PLATFORMS, test=True))
-        self.assertEqual(build_plan(core_changed=False, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=True))
-
-    def test_core_change_leaves_tests_to_build_yml(self):
-        self.assertEqual(build_plan(core_changed=True, addon_changed=False), BuildPlan([], test=False))
-        self.assertEqual(build_plan(core_changed=True, addon_changed=True), BuildPlan(ADDON_PLATFORMS, test=False))
+    def test_reads_the_pull_request_paths_of_build_yml(self):
+        paths = pull_request_paths((packages_release.ROOT / BUILD_WORKFLOW).read_text())
+        self.assertIn("src/**", paths)
+        self.assertIn(".github/**", paths)
+        self.assertFalse([path for path in paths if path.startswith("packages/")])
 
     def test_matrix_names_the_runner_of_each_platform(self):
         self.assertEqual(
@@ -406,6 +405,40 @@ class GitRepoTest(unittest.TestCase):
         ).stdout.strip()
 
 
+class PrintBuildPlanTest(GitRepoTest):
+    def setUp(self):
+        super().setUp()
+        self.write(BUILD_WORKFLOW, 'on:\n  pull_request:\n    paths:\n      - "src/**"\n      - "Dockerfile*"\n\nenv:\n')
+        self.write("packages/pkg/src.txt", "a")
+        self.commit("base")
+        self.base = self.head()
+
+    def platforms(self):
+        output = io.StringIO()
+        with mock.patch.object(packages_release, "ROOT", self.repo), contextlib.redirect_stdout(output):
+            print_build_plan(argparse.Namespace(base=self.base))
+        return [entry["platform"] for entry in json.loads(output.getvalue().removeprefix("platforms="))]
+
+    def test_package_change_is_tested_on_every_platform(self):
+        self.write("packages/pkg/src.txt", "b")
+        self.commit("package change")
+        self.assertEqual(self.platforms(), ADDON_PLATFORMS)
+
+    def test_change_that_runs_build_yml_is_tested_there(self):
+        self.write("packages/pkg/src.txt", "b")
+        self.write("src/Lib/Module.hs", "m")
+        self.commit("package and core change")
+        self.assertEqual(self.platforms(), [])
+
+    def test_patterns_match_as_path_filters(self):
+        self.write("packages/pkg/Dockerfile", "d")
+        self.commit("nested Dockerfile")
+        self.assertEqual(self.platforms(), ADDON_PLATFORMS)
+        self.write("Dockerfile.build", "d")
+        self.commit("top-level Dockerfile")
+        self.assertEqual(self.platforms(), [])
+
+
 class PackageStateTest(GitRepoTest):
     def setUp(self):
         super().setUp()