From 9fc90ee099be3df2aa8ac381e1392d3928187fcd Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 19 Jul 2026 17:23:10 -0700 Subject: [PATCH] ci(beta): retry GitHub API calls in the publish step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 29708863985 had every build shard green and then died here: HTTP 503 ... (https://api.github.com/repos/agessaman/MeshCore/releases) Error: Process completed with exit code 1 'gh release create' hit a transient 503 during a GitHub incident and, under 'bash -e', threw away ~15 minutes of building across 14 runners. Nothing was wrong with the code. Add a gh_retry helper (5 attempts, exponential backoff 10/20/40/80s) around the create and upload calls. 'until' in a condition does not trip -e, so the helper is safe in this shell. Deliberate choices: - The existence check is NOT retried: 'release does not exist' is the expected answer on a first run and retrying would only burn backoff. A 5xx there falls through to create, which now tolerates an already-existing release. - Prune failures no longer fail the job. Pruning is housekeeping that runs AFTER a successful upload; leaving stale assets until the next run beats reporting failure for a build whose binaries are already published. - The prune's second 'gh release view' is gone — it reuses the asset list already fetched, removing an API call as well as an unretried failure point. Retry helper unit-tested for the success, transient-recovery, and exhaustion paths. Production (build-observer-firmwares.yml) has the identical fragility and should get the same treatment; not changed here to keep this scoped to the beta channel. --- .../build-observer-firmwares-beta.yml | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-observer-firmwares-beta.yml b/.github/workflows/build-observer-firmwares-beta.yml index 4a7c02d8..3a41acc4 100644 --- a/.github/workflows/build-observer-firmwares-beta.yml +++ b/.github/workflows/build-observer-firmwares-beta.yml @@ -208,24 +208,60 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | + # Retry wrapper for GitHub API calls. This job runs AFTER ~15 minutes of + # building across 14 runners, and every call below is an API write; with + # `bash -e`, a single transient 5xx throws all of that away. Observed + # 2026-07-19: `gh release create` got HTTP 503 during a GitHub incident + # and killed a run whose builds had all passed. + # `until` in a condition does not trip `-e`, so this is safe here. + gh_retry() { + local n=0 max=5 delay=10 + until "$@"; do + n=$((n + 1)) + if [ "$n" -ge "$max" ]; then + echo "::error::gh failed after $max attempts: $*" >&2 + return 1 + fi + echo "gh call failed (attempt $n/$max), retrying in ${delay}s: $*" >&2 + sleep "$delay" + delay=$((delay * 2)) + done + } + + # Deliberately NOT retried: a plain "release does not exist" is the + # expected answer on the first run, and retrying it would just burn the + # backoff. A 5xx here instead makes us fall through to create, which is + # then tolerated below if the release actually did already exist. if ! gh release view "$RELEASE_TAG" >/dev/null 2>&1; then - gh release create "$RELEASE_TAG" --prerelease \ + gh_retry gh release create "$RELEASE_TAG" --prerelease \ --title "MQTT Observer Firmwares (BETA)" \ - --notes "Rolling BETA build. Separate channel from observer-mqtt-latest; beta nodes only OTA within this channel." + --notes "Rolling BETA build. Separate channel from observer-mqtt-latest; beta nodes only OTA within this channel." \ + || gh release view "$RELEASE_TAG" >/dev/null 2>&1 \ + || { echo "::error::could not create or confirm $RELEASE_TAG" >&2; exit 1; } fi - gh release upload "$RELEASE_TAG" $(find out -maxdepth 1 -type f ! -name '*.partsig') --clobber + gh_retry gh release upload "$RELEASE_TAG" $(find out -maxdepth 1 -type f ! -name '*.partsig') --clobber + # Pruning is housekeeping and runs AFTER the upload has succeeded. If the + # API is flaky here, skip it rather than fail the job — old assets simply + # linger until the next run, which is strictly better than reporting + # failure for a build whose binaries are already published. KEEP_BUILDS=2 - keep_hashes=$(gh release view "$RELEASE_TAG" --json assets \ - -q '.assets[] | "\(.createdAt) \(.name)"' \ + if ! asset_list=$(gh_retry gh release view "$RELEASE_TAG" --json assets \ + -q '.assets[] | "\(.createdAt) \(.name)"'); then + echo "::warning::could not list assets; skipping prune this run" + exit 0 + fi + keep_hashes=$(printf '%s\n' "$asset_list" \ | sort -r \ | while read -r _ts name; do printf '%s' "$name" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' done \ | awk '!seen[$0]++' | head -n "$KEEP_BUILDS") echo "Retaining build hashes:"; echo "$keep_hashes" - gh release view "$RELEASE_TAG" --json assets -q '.assets[].name' \ + # Reuse asset_list rather than making a second API call (its lines are + # " ", so the name is field 2). + printf '%s\n' "$asset_list" | awk '{print $2}' \ | while read -r asset; do ah=$(printf '%s' "$asset" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' || true) if [ -n "$ah" ] && grep -qxF "$ah" <<<"$keep_hashes"; then