Hide successful, skipped Complement tests in the CI (#19590)

Co-authored-by: Eric Eastwood <erice@element.io>

.ci/complement_package.gotpl

@@ -29,7 +29,7 @@ which is under the Unlicense licence.
     {{- with .TestCases -}}
         {{- /* Passing tests are first */ -}}
         {{- range . -}}
-            {{- if eq .Result "PASS" -}}
+            {{- if and (eq .Result "PASS") (not $settings.HideSuccessfulTests) -}}
                 ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}}
                 {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
                 {{- with .Coverage -}}
@@ -49,7 +49,8 @@ which is under the Unlicense licence.
 
         {{- /* Then skipped tests are second */ -}}
         {{- range . -}}
-            {{- if eq .Result "SKIP" -}}
+            {{- /* Skipped tests are also purposefully hidden if "HideSuccessfulTests" is enabled */ -}}
+            {{- if and (eq .Result "SKIP") (not $settings.HideSuccessfulTests) -}}
                 ::group::{{ "\033" }}[0;33m🚧{{ " " }}{{- .Name -}}
                 {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
                 {{- with .Coverage -}}

.ci/scripts/gotestfmt

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# wraps `gotestfmt`, hiding output from successful packages unless
-# all tests passed.
-
-set -o pipefail
-set -e
-
-# tee the test results to a log, whilst also piping them into gotestfmt,
-# telling it to hide successful results, so that we can clearly see
-# unsuccessful results.
-tee complement.log | gotestfmt -hide successful-packages
-
-# gotestfmt will exit non-zero if there were any failures, so if we got to this
-# point, we must have had a successful result.
-echo "All tests successful; showing all test results"
-
-# Pipe the test results back through gotestfmt, showing all results.
-# The log file consists of JSON lines giving the test results, interspersed
-# with regular stdout lines (including reports of downloaded packages).
-grep '^{"Time":' complement.log | gotestfmt

.ci/scripts/setup_complement_prerequisites.sh

@@ -10,7 +10,6 @@ alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; fu
 alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'
 
 block Install Complement Dependencies
-  sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
   go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
 endblock
 

.github/workflows/tests.yml

@@ -744,6 +744,12 @@ jobs:
       - name: Formatted sanity check Complement test logs
         # Always run this step if we attempted to run the Complement tests.
         if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped'
+        # We do not hide successful tests in `gotestfmt` here as the list of sanity
+        # check tests is so short. Feel free to change this when we get more tests.
+        #
+        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
+        # it derives several values under `$settings` and passes them to our
+        # custom `.ci/complement_package.gotpl` template to render the output.
         run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
 
       - name: Run Complement Tests
@@ -764,10 +770,15 @@ jobs:
           POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
           WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
 
-      - name: Formatted Complement test logs
+      - name: Formatted Complement test logs (only failing are shown)
         # Always run this step if we attempted to run the Complement tests.
         if: always() && steps.run_complement_tests.outcome != 'skipped'
-        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
+        # Hide successful tests in order to reduce the verbosity of the otherwise very large output.
+        #
+        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
+        # it derives several values under `$settings` and passes them to our
+        # custom `.ci/complement_package.gotpl` template to render the output.
+        run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages"
 
       - name: Run in-repo Complement Tests
         id: run_in_repo_complement_tests
@@ -787,10 +798,15 @@ jobs:
           POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
           WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
 
-      - name: Formatted in-repo Complement test logs
+      - name: Formatted in-repo Complement test logs (only failing are shown)
         # Always run this step if we attempted to run the Complement tests.
         if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped'
-        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages"
+        # Hide successful tests in order to reduce the verbosity of the otherwise very large output.
+        #
+        # Note that the `-hide` argument is interpreted by `gotestfmt`. From it,
+        # it derives several values under `$settings` and passes them to our
+        # custom `.ci/complement_package.gotpl` template to render the output.
+        run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages"
 
   cargo-test:
     if: ${{ needs.changes.outputs.rust == 'true' }}

changelog.d/19590.misc

@@ -0,0 +1 @@
+Only show failing Complement tests in the formatted output in CI.