From 3dd68d44183b3722fe93f80d91e86e647b897073 Mon Sep 17 00:00:00 2001
From: you <you@example.com>
Date: Sun, 29 Mar 2026 23:16:46 +0000
Subject: [PATCH] =?UTF-8?q?fix:=20staging=20deploy=20failures=20=E2=80=94?=
 =?UTF-8?q?=20OOM=20+=20config.json=20directory=20mount?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Root causes from CI logs:
1. 'read /app/config.json: is a directory' — Docker creates a directory
   when bind-mounting a non-existent file. The entrypoint now detects
   and removes directory config.json before falling back to example.
2. 'unable to open database file: out of memory (14)' — old container
   (3GB) not fully exited when new one starts. Deploy now uses
   'docker compose down' with timeout and waits for memory reclaim.
3. Supervisor gave up after 3 fast retries (FATAL in ~6s). Increased
   startretries to 10 and startsecs to 2 for server and ingestor.

Additional:
- Deploy step ensures staging config.json exists before starting
- Healthcheck: added start_period=60s, increased timeout and retries
- No longer uses manage.sh (CI working dir != repo checkout dir)
---
 .github/workflows/deploy.yml | 24 +++++++++++++++++++++++-
 docker-compose.staging.yml   |  5 +++--
 docker/entrypoint-go.sh      |  7 +++++++
 docker/supervisord-go.conf   |  4 ++++
 4 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 92c2bbbb..667d2af6 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -262,7 +262,29 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Deploy staging
-        run: ./manage.sh restart staging
+        run: |
+          # Use docker compose down (not just stop/rm) to properly clean up
+          # the old container, network, and release memory before starting new one
+          docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging down --timeout 30 2>/dev/null || true
+
+          # Wait for container to be fully gone and OS to reclaim memory (3GB limit)
+          for i in $(seq 1 15); do
+            if ! docker ps -a --format '{{.Names}}' | grep -q 'corescope-staging-go'; then
+              break
+            fi
+            sleep 1
+          done
+          sleep 5  # extra pause for OS memory reclaim
+
+          # Ensure staging config exists (docker creates a directory if bind mount source missing)
+          STAGING_DATA="${STAGING_DATA_DIR:-$HOME/meshcore-staging-data}"
+          if [ ! -f "$STAGING_DATA/config.json" ]; then
+            echo "Staging config missing — copying from repo config.example.json"
+            mkdir -p "$STAGING_DATA"
+            cp config.example.json "$STAGING_DATA/config.json"
+          fi
+
+          docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d staging-go
 
       - name: Healthcheck staging container
         run: |
diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml
index bb682f5e..e2fd19ae 100644
--- a/docker-compose.staging.yml
+++ b/docker-compose.staging.yml
@@ -34,8 +34,9 @@ services:
     healthcheck:
       test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"]
       interval: 30s
-      timeout: 5s
-      retries: 3
+      timeout: 10s
+      retries: 5
+      start_period: 60s
 
 volumes:
   # Named volume for Caddy TLS certificates (not user data — managed by Caddy internally)
diff --git a/docker/entrypoint-go.sh b/docker/entrypoint-go.sh
index 752b34c0..36dad083 100644
--- a/docker/entrypoint-go.sh
+++ b/docker/entrypoint-go.sh
@@ -1,5 +1,12 @@
 #!/bin/sh
 
+# Fix: Docker creates a directory when bind-mounting a non-existent file.
+# If config.json is a directory (from a failed mount), remove it and use the example.
+if [ -d /app/config.json ]; then
+  echo "[entrypoint] WARNING: config.json is a directory (broken bind mount) — removing and using example"
+  rm -rf /app/config.json
+fi
+
 # Copy example config if no config.json exists (not bind-mounted)
 if [ ! -f /app/config.json ]; then
   echo "[entrypoint] No config.json found, copying from config.example.json"
diff --git a/docker/supervisord-go.conf b/docker/supervisord-go.conf
index 8c783615..76f53c05 100644
--- a/docker/supervisord-go.conf
+++ b/docker/supervisord-go.conf
@@ -19,6 +19,8 @@ command=/app/corescope-ingestor -config /app/config.json
 directory=/app
 autostart=true
 autorestart=true
+startretries=10
+startsecs=2
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
@@ -29,6 +31,8 @@ command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public
 directory=/app
 autostart=true
 autorestart=true
+startretries=10
+startsecs=2
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr