name: Deploy on: push: branches: [master] paths-ignore: - '**.md' - 'LICENSE' - '.gitignore' - 'docs/**' concurrency: group: deploy cancel-in-progress: true env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # Pipeline (TWO INDEPENDENT TRACKS): # Track 1 (Node): node-test → build-node → deploy-node ──┐ # Track 2 (Go): go-test → build-go → deploy-go ──┼──→ publish # └─ (both wait) jobs: # ─────────────────────────────────────────────────────────────── # 1. Go Build & Test — compiles + tests Go modules, coverage badges # ─────────────────────────────────────────────────────────────── go-test: name: "✅ Go Build & Test" runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Go 1.22 uses: actions/setup-go@v5 with: go-version: '1.22' cache-dependency-path: | cmd/server/go.sum cmd/ingestor/go.sum - name: Build and test Go server (with coverage) run: | cd cmd/server go build . go test -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log echo "--- Go Server Coverage ---" go tool cover -func=server-coverage.out | tail -1 - name: Build and test Go ingestor (with coverage) run: | cd cmd/ingestor go build . go test -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log echo "--- Go Ingestor Coverage ---" go tool cover -func=ingestor-coverage.out | tail -1 - name: Generate Go coverage badges if: always() run: | mkdir -p .badges # Parse server coverage SERVER_COV="0" if [ -f cmd/server/server-coverage.out ]; then SERVER_COV=$(cd cmd/server && go tool cover -func=server-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)') fi SERVER_COLOR="red" if [ "$(echo "$SERVER_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then SERVER_COLOR="green" elif [ "$(echo "$SERVER_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then SERVER_COLOR="yellow" fi echo "{\"schemaVersion\":1,\"label\":\"go server coverage\",\"message\":\"${SERVER_COV}%\",\"color\":\"${SERVER_COLOR}\"}" > .badges/go-server-coverage.json echo "Go server coverage: ${SERVER_COV}% (${SERVER_COLOR})" # Parse ingestor coverage INGESTOR_COV="0" if [ -f cmd/ingestor/ingestor-coverage.out ]; then INGESTOR_COV=$(cd cmd/ingestor && go tool cover -func=ingestor-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)') fi INGESTOR_COLOR="red" if [ "$(echo "$INGESTOR_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then INGESTOR_COLOR="green" elif [ "$(echo "$INGESTOR_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then INGESTOR_COLOR="yellow" fi echo "{\"schemaVersion\":1,\"label\":\"go ingestor coverage\",\"message\":\"${INGESTOR_COV}%\",\"color\":\"${INGESTOR_COLOR}\"}" > .badges/go-ingestor-coverage.json echo "Go ingestor coverage: ${INGESTOR_COV}% (${INGESTOR_COLOR})" echo "## Go Coverage" >> $GITHUB_STEP_SUMMARY echo "| Module | Coverage |" >> $GITHUB_STEP_SUMMARY echo "|--------|----------|" >> $GITHUB_STEP_SUMMARY echo "| Server | ${SERVER_COV}% |" >> $GITHUB_STEP_SUMMARY echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY - name: Upload Go coverage badges if: always() uses: actions/upload-artifact@v4 with: name: go-badges path: .badges/go-*.json retention-days: 1 if-no-files-found: ignore # ─────────────────────────────────────────────────────────────── # 2. Node.js Tests — backend unit tests + Playwright E2E, coverage # ─────────────────────────────────────────────────────────────── node-test: name: "🧪 Node.js Tests" runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 2 - name: Set up Node.js 22 uses: actions/setup-node@v4 with: node-version: '22' - name: Install npm dependencies run: npm ci --production=false - name: Detect changed files id: changes run: | BACKEND=$(git diff --name-only HEAD~1 | grep -cE '^(server|db|decoder|packet-store|server-helpers|iata-coords)\.js$' || true) FRONTEND=$(git diff --name-only HEAD~1 | grep -cE '^public/' || true) TESTS=$(git diff --name-only HEAD~1 | grep -cE '^test-|^tools/' || true) CI=$(git diff --name-only HEAD~1 | grep -cE '\.github/|package\.json|test-all\.sh|scripts/' || true) # If CI/test infra changed, run everything if [ "$CI" -gt 0 ]; then BACKEND=1; FRONTEND=1; fi # If test files changed, run everything if [ "$TESTS" -gt 0 ]; then BACKEND=1; FRONTEND=1; fi echo "backend=$([[ $BACKEND -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT echo "frontend=$([[ $FRONTEND -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT echo "Changes: backend=$BACKEND frontend=$FRONTEND tests=$TESTS ci=$CI" - name: Run backend tests with coverage if: steps.changes.outputs.backend == 'true' run: | npx c8 --reporter=text-summary --reporter=text sh test-all.sh 2>&1 | tee test-output.txt TOTAL_PASS=$(grep -oP '\d+(?= passed)' test-output.txt | awk '{s+=$1} END {print s}') TOTAL_FAIL=$(grep -oP '\d+(?= failed)' test-output.txt | awk '{s+=$1} END {print s}') BE_COVERAGE=$(grep 'Statements' test-output.txt | tail -1 | grep -oP '[\d.]+(?=%)') mkdir -p .badges BE_COLOR="red" [ "$(echo "$BE_COVERAGE > 60" | bc -l 2>/dev/null)" = "1" ] && BE_COLOR="yellow" [ "$(echo "$BE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && BE_COLOR="brightgreen" echo "{\"schemaVersion\":1,\"label\":\"backend tests\",\"message\":\"${TOTAL_PASS} passed\",\"color\":\"brightgreen\"}" > .badges/backend-tests.json echo "{\"schemaVersion\":1,\"label\":\"backend coverage\",\"message\":\"${BE_COVERAGE}%\",\"color\":\"${BE_COLOR}\"}" > .badges/backend-coverage.json echo "## Backend: ${TOTAL_PASS} tests, ${BE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY - name: Run backend tests (quick, no coverage) if: steps.changes.outputs.backend == 'false' run: npm run test:unit - name: Install Playwright browser if: steps.changes.outputs.frontend == 'true' run: npx playwright install chromium --with-deps 2>/dev/null || true - name: Instrument frontend JS for coverage if: steps.changes.outputs.frontend == 'true' run: sh scripts/instrument-frontend.sh - name: Start instrumented test server on port 13581 if: steps.changes.outputs.frontend == 'true' run: | COVERAGE=1 PORT=13581 node server.js & echo $! > .server.pid echo "Server PID: $(cat .server.pid)" # Health-check poll loop (up to 30s) for i in $(seq 1 30); do if curl -sf http://localhost:13581/api/stats > /dev/null 2>&1; then echo "Server ready after ${i}s" break fi if [ "$i" -eq 30 ]; then echo "Server failed to start within 30s" exit 1 fi sleep 1 done - name: Run Playwright E2E tests if: steps.changes.outputs.frontend == 'true' run: BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt - name: Collect frontend coverage report if: always() && steps.changes.outputs.frontend == 'true' run: | BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt E2E_PASS=$(grep -oP '[0-9]+(?=/)' e2e-output.txt | tail -1) mkdir -p .badges if [ -f .nyc_output/frontend-coverage.json ]; then npx nyc report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt FE_COVERAGE=$(grep 'Statements' fe-report.txt | head -1 | grep -oP '[\d.]+(?=%)' || echo "0") FE_COVERAGE=${FE_COVERAGE:-0} FE_COLOR="red" [ "$(echo "$FE_COVERAGE > 50" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="yellow" [ "$(echo "$FE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="brightgreen" echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY fi echo "{\"schemaVersion\":1,\"label\":\"frontend tests\",\"message\":\"${E2E_PASS:-0} E2E passed\",\"color\":\"brightgreen\"}" > .badges/frontend-tests.json - name: Stop test server if: always() && steps.changes.outputs.frontend == 'true' run: | if [ -f .server.pid ]; then kill $(cat .server.pid) 2>/dev/null || true rm -f .server.pid echo "Server stopped" fi - name: Run frontend E2E (quick, no coverage) if: steps.changes.outputs.frontend == 'false' run: | fuser -k 13581/tcp 2>/dev/null || true PORT=13581 node server.js & SERVER_PID=$! sleep 5 BASE_URL=http://localhost:13581 node test-e2e-playwright.js || true kill $SERVER_PID 2>/dev/null || true - name: Upload Node.js test badges if: always() uses: actions/upload-artifact@v4 with: name: node-badges path: .badges/ retention-days: 1 if-no-files-found: ignore # ─────────────────────────────────────────────────────────────── # 3. Build Node Docker Image — Track 1 # ─────────────────────────────────────────────────────────────── build-node: name: "🏗️ Build Node Docker Image" needs: [node-test] runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Node.js 22 uses: actions/setup-node@v4 with: node-version: '22' - name: Validate JavaScript syntax run: sh scripts/validate.sh - name: Build Node.js Docker image run: | echo "${GITHUB_SHA::7}" > .git-commit docker build -t meshcore-analyzer:latest . echo "Built meshcore-analyzer:latest ($(git rev-parse --short HEAD))" # ─────────────────────────────────────────────────────────────── # 4. Build Go Docker Image — Track 2 # ─────────────────────────────────────────────────────────────── build-go: name: "🏗️ Build Go Docker Image" needs: [go-test] runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Node.js 22 uses: actions/setup-node@v4 with: node-version: '22' - name: Build Go Docker image run: | echo "${GITHUB_SHA::7}" > .git-commit APP_VERSION=$(node -p "require('./package.json').version") \ GIT_COMMIT="${GITHUB_SHA::7}" \ docker compose --profile staging-go build staging-go echo "Built Go staging image" # ─────────────────────────────────────────────────────────────── # 5. Deploy Node Staging — start on port 81, healthcheck, smoke test # ─────────────────────────────────────────────────────────────── deploy-node: name: "🚀 Deploy Node Staging" needs: [build-node] runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v4 - name: Prepare staging environment run: | set -e # Source environment overrides from deploy dir or home if [ -f /opt/meshcore-deploy/.env ]; then set -a; source /opt/meshcore-deploy/.env; set +a echo "Sourced /opt/meshcore-deploy/.env" elif [ -f "$HOME/.env" ]; then set -a; source "$HOME/.env"; set +a echo "Sourced $HOME/.env" else echo "No .env found, using defaults" fi # Ensure data directories exist STAGING_DIR="${STAGING_DATA_DIR:-$HOME/meshcore-staging-data}" mkdir -p "${PROD_DATA_DIR:-$HOME/meshcore-data}" "$STAGING_DIR" # Ensure staging has a Caddyfile (generate from template if missing) if [ ! -f "$STAGING_DIR/Caddyfile" ]; then cp docker/Caddyfile.staging "$STAGING_DIR/Caddyfile" echo "Generated staging Caddyfile" fi # Ensure staging has a config.json (copy from prod if missing) if [ ! -f "$STAGING_DIR/config.json" ]; then PROD_DIR="${PROD_DATA_DIR:-$HOME/meshcore-data}" if [ -f "$PROD_DIR/config.json" ]; then cp "$PROD_DIR/config.json" "$STAGING_DIR/config.json" elif [ -f /opt/meshcore-deploy/config.json ]; then cp /opt/meshcore-deploy/config.json "$STAGING_DIR/config.json" else echo '{}' > "$STAGING_DIR/config.json" echo "WARNING: No config.json found, created empty one" fi fi # Copy compose file to deploy dir so manage.sh can use it mkdir -p /opt/meshcore-deploy cp docker-compose.yml /opt/meshcore-deploy/docker-compose.yml - name: Start Node staging on port 81 run: | docker rm -f meshcore-staging 2>/dev/null || true docker compose --profile staging up -d staging - name: Healthcheck Node staging container run: | for i in $(seq 1 300); do HEALTH=$(docker inspect meshcore-staging --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") if [ "$HEALTH" = "healthy" ]; then echo "Node staging healthy after ${i}s" break fi if [ "$i" -eq 300 ]; then echo "Node staging failed health check after 300s" docker logs meshcore-staging --tail 50 exit 1 fi sleep 1 done - name: Smoke test Node staging API run: | curl -f http://localhost:81/api/stats || exit 1 curl -f http://localhost:81/api/nodes || exit 1 echo "Node staging smoke tests passed ✅" - name: Validate API contract (proto vs Node fixtures) run: | set -e echo "Refreshing Node fixtures from staging container..." mkdir -p proto/testdata/node-fixtures # ─── Simple endpoints (no parameters) ────────────────────────── ENDPOINTS=( "stats" "health" "perf" "nodes" "packets" "observers" "channels" "analytics/rf" "analytics/topology" "analytics/channels" "analytics/hash-sizes" "analytics/distance" "analytics/subpaths" "config/theme" "config/regions" "config/client" "config/cache" "config/map" "iata-coords" ) for endpoint in "${ENDPOINTS[@]}"; do fixture_name=$(echo "$endpoint" | tr '/' '-') echo " Fetching $endpoint → ${fixture_name}.json" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/$endpoint" \ > "proto/testdata/node-fixtures/${fixture_name}.json" 2>/dev/null || { echo " ⚠ Failed to fetch $endpoint (container may not have data yet)" } done # ─── Dynamic ID endpoints (require real data) ───────────────── echo "" echo "Fetching endpoints that require dynamic IDs..." # Get a real pubkey from nodes PUBKEY=$(docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes?limit=1" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['nodes'][0]['public_key'] if d.get('nodes') and len(d['nodes']) > 0 else '')" 2>/dev/null || echo "") if [ -n "$PUBKEY" ]; then echo " Using pubkey: ${PUBKEY:0:16}..." docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes/$PUBKEY" \ > "proto/testdata/node-fixtures/node-detail.json" 2>/dev/null && \ echo " ✓ node-detail.json" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes/$PUBKEY/health" \ > "proto/testdata/node-fixtures/node-health.json" 2>/dev/null && \ echo " ✓ node-health.json" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes/$PUBKEY/paths" \ > "proto/testdata/node-fixtures/node-paths.json" 2>/dev/null && \ echo " ✓ node-paths.json" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes/$PUBKEY/analytics" \ > "proto/testdata/node-fixtures/node-analytics.json" 2>/dev/null && \ echo " ✓ node-analytics.json" else echo " ⚠ No nodes available — skipping node detail endpoints" fi # Node search docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes/search?q=repeater" \ > "proto/testdata/node-fixtures/node-search.json" 2>/dev/null && \ echo " ✓ node-search.json" || echo " ⚠ node-search failed" # Bulk health docker exec meshcore-prod wget -qO- "http://localhost:3000/api/nodes/bulk-health" \ > "proto/testdata/node-fixtures/bulk-health.json" 2>/dev/null && \ echo " ✓ bulk-health.json" || echo " ⚠ bulk-health failed" # Get a real hash from packets HASH=$(docker exec meshcore-prod wget -qO- "http://localhost:3000/api/packets?limit=1" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['packets'][0]['hash'] if d.get('packets') and len(d['packets']) > 0 else '')" 2>/dev/null || echo "") if [ -n "$HASH" ]; then echo " Using hash: $HASH" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/packets/$HASH" \ > "proto/testdata/node-fixtures/packet-detail.json" 2>/dev/null && \ echo " ✓ packet-detail.json" else echo " ⚠ No packets available — skipping packet-detail" fi # Packet timestamps docker exec meshcore-prod wget -qO- "http://localhost:3000/api/packets/timestamps?since=2026-03-01T00:00:00Z" \ > "proto/testdata/node-fixtures/packet-timestamps.json" 2>/dev/null && \ echo " ✓ packet-timestamps.json" || echo " ⚠ packet-timestamps failed" # Packets grouped and since docker exec meshcore-prod wget -qO- "http://localhost:3000/api/packets?limit=5&groupByHash=true" \ > "proto/testdata/node-fixtures/packets-grouped.json" 2>/dev/null && \ echo " ✓ packets-grouped.json" || echo " ⚠ packets-grouped failed" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/packets?limit=5&since=2026-03-01T00:00:00Z&groupByHash=true" \ > "proto/testdata/node-fixtures/packets-since.json" 2>/dev/null && \ echo " ✓ packets-since.json" || echo " ⚠ packets-since failed" # Get a real observer ID OBSID=$(docker exec meshcore-prod wget -qO- "http://localhost:3000/api/observers" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['id'] if d and len(d) > 0 else '')" 2>/dev/null || echo "") if [ -n "$OBSID" ]; then echo " Using observer ID: $OBSID" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/observers/$OBSID" \ > "proto/testdata/node-fixtures/observer-detail.json" 2>/dev/null && \ echo " ✓ observer-detail.json" docker exec meshcore-prod wget -qO- "http://localhost:3000/api/observers/$OBSID/analytics" \ > "proto/testdata/node-fixtures/observer-analytics.json" 2>/dev/null && \ echo " ✓ observer-analytics.json" else echo " ⚠ No observers available — skipping observer detail endpoints" fi # Channel messages docker exec meshcore-prod wget -qO- "http://localhost:3000/api/channels/public/messages?limit=5" \ > "proto/testdata/node-fixtures/channel-messages.json" 2>/dev/null && \ echo " ✓ channel-messages.json" || echo " ⚠ channel-messages failed" # WebSocket message capture (capture one message if available) echo " Capturing WebSocket message..." docker exec meshcore-prod node -e " const WebSocket = require('ws'); const ws = new WebSocket('ws://localhost:3000'); const timeout = setTimeout(() => { console.error('timeout'); process.exit(1); }, 5000); ws.on('message', (data) => { console.log(data); clearTimeout(timeout); ws.close(); process.exit(0); }); ws.on('error', () => { clearTimeout(timeout); process.exit(1); }); " > "proto/testdata/node-fixtures/websocket-message.json" 2>/dev/null && \ echo " ✓ websocket-message.json" || echo " ⚠ websocket-message failed (no live packets)" echo "" echo "Running proto validator..." python3 tools/validate-protos.py || { echo "❌ Proto validation failed — API contract drift detected" echo "This means a Node.js API response doesn't match the proto definition." echo "Fix by updating the .proto files in proto/ to match the actual API responses." exit 1 } echo "✅ Proto validation passed — API contract is consistent" # ─────────────────────────────────────────────────────────────── # 6. Deploy Go Staging — start on port 82, healthcheck, smoke test # ─────────────────────────────────────────────────────────────── deploy-go: name: "🚀 Deploy Go Staging" needs: [build-go] runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v4 - name: Start Go staging on port 82 run: | docker rm -f meshcore-staging-go 2>/dev/null || true docker compose --profile staging-go up -d staging-go - name: Healthcheck Go staging container run: | for i in $(seq 1 60); do HEALTH=$(docker inspect meshcore-staging-go --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") if [ "$HEALTH" = "healthy" ]; then echo "Go staging healthy after ${i}s" break fi if [ "$i" -eq 60 ]; then echo "Go staging failed health check after 60s" docker logs meshcore-staging-go --tail 30 exit 1 fi sleep 1 done - name: Smoke test Go staging API run: | if curl -sf http://localhost:82/api/stats | grep -q engine; then echo "Go staging verified — engine field present ✅" else echo "Go staging /api/stats did not return engine field" exit 1 fi # ─────────────────────────────────────────────────────────────── # 7. Publish Badges & Summary — waits for both tracks to complete # ─────────────────────────────────────────────────────────────── publish: name: "📝 Publish Badges & Summary" needs: [deploy-node, deploy-go] runs-on: self-hosted steps: - name: Checkout code uses: actions/checkout@v4 - name: Download Go coverage badges continue-on-error: true uses: actions/download-artifact@v4 with: name: go-badges path: .badges/ - name: Download Node.js test badges continue-on-error: true uses: actions/download-artifact@v4 with: name: node-badges path: .badges/ - name: Publish coverage badges to repo continue-on-error: true run: | git config user.name "github-actions" git config user.email "actions@github.com" git remote set-url origin https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git git add .badges/ -f git diff --cached --quiet || (git commit -m "ci: update test badges [skip ci]" && git push) || echo "Badge push failed" - name: Post deployment summary run: | echo "## Staging Deployed ✓" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Commit:** \`$(git rev-parse --short HEAD)\` — $(git log -1 --format=%s)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Node Staging:** http://:81" >> $GITHUB_STEP_SUMMARY echo "**Go Staging:** http://:82" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "To promote to production:" >> $GITHUB_STEP_SUMMARY echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY echo "ssh deploy@" >> $GITHUB_STEP_SUMMARY echo "cd /opt/meshcore-deploy" >> $GITHUB_STEP_SUMMARY echo "./manage.sh promote" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY