From 067b101e14427a72564fa0220b0e37d6b3637316 Mon Sep 17 00:00:00 2001 From: Kpa-clawbot <259247574+Kpa-clawbot@users.noreply.github.com> Date: Sun, 29 Mar 2026 13:20:41 -0700 Subject: [PATCH 1/2] fix: split prod/staging compose and harden deploy/manage staging control Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/deploy.yml | 21 +++--- docker-compose.staging.yml | 38 ++++++++++ docker-compose.yml | 139 ++++++++++------------------------- manage.sh | 49 +++++++----- 4 files changed, 115 insertions(+), 132 deletions(-) create mode 100644 docker-compose.staging.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fdae50e2..cea04497 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,6 +12,9 @@ concurrency: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + STAGING_COMPOSE_FILE: docker-compose.staging.yml + STAGING_SERVICE: staging-go + STAGING_CONTAINER: corescope-staging-go # Pipeline (sequential, fail-fast): # go-test → e2e-test → build → deploy → publish @@ -243,7 +246,7 @@ jobs: GIT_COMMIT=$(git rev-parse --short HEAD) BUILD_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ') export APP_VERSION GIT_COMMIT BUILD_TIME - docker compose --profile staging-go build staging-go + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging build "$STAGING_SERVICE" echo "Built Go staging image ✅" # ─────────────────────────────────────────────────────────────── @@ -258,28 +261,24 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - - name: Stop old containers - run: | - docker compose --profile staging-go down 2>/dev/null || true - # Kill legacy container names from pre-rename era - docker rm -f meshcore-staging meshcore-staging-go corescope-staging 2>/dev/null || true - sleep 2 + - name: Stop old staging + run: ./manage.sh stop staging - - name: Start staging on port 82 + - name: Start staging run: | - docker compose --profile staging-go up -d staging-go + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d "$STAGING_SERVICE" - name: Healthcheck staging container run: | for i in $(seq 1 120); do - HEALTH=$(docker inspect corescope-staging-go --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") + HEALTH=$(docker inspect "$STAGING_CONTAINER" --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") if [ "$HEALTH" = "healthy" ]; then echo "Staging healthy after ${i}s" break fi if [ "$i" -eq 120 ]; then echo "Staging failed health check after 120s" - docker logs corescope-staging-go --tail 50 + docker logs "$STAGING_CONTAINER" --tail 50 exit 1 fi sleep 1 diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml new file mode 100644 index 00000000..0681bf46 --- /dev/null +++ b/docker-compose.staging.yml @@ -0,0 +1,38 @@ +# Staging-only compose file. Production is managed by docker-compose.yml. +# Override defaults via .env or environment variables. + +services: + staging-go: + build: + context: . + dockerfile: Dockerfile + args: + APP_VERSION: ${APP_VERSION:-unknown} + GIT_COMMIT: ${GIT_COMMIT:-unknown} + BUILD_TIME: ${BUILD_TIME:-unknown} + image: corescope-go:latest + container_name: corescope-staging-go + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "${STAGING_GO_HTTP_PORT:-82}:80" + - "${STAGING_GO_MQTT_PORT:-1885}:1883" + - "6060:6060" # pprof server + - "6061:6061" # pprof ingestor + volumes: + - ${STAGING_DATA_DIR:-~/meshcore-staging-data}/config.json:/app/config.json:ro + - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data + - caddy-data-staging-go:/data/caddy + environment: + - NODE_ENV=staging + - ENABLE_PPROF=true + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + interval: 30s + timeout: 5s + retries: 3 + +volumes: + # Named volume for Caddy TLS certificates (not user data — managed by Caddy internally) + caddy-data-staging-go: diff --git a/docker-compose.yml b/docker-compose.yml index 68d7742e..b0f73ea0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,103 +1,38 @@ -# All container config lives here. manage.sh is just a wrapper around docker compose. -# Override defaults via .env or environment variables. -# CRITICAL: All data mounts use bind mounts (~/path), NOT named volumes. -# This ensures the DB and theme are visible on the host filesystem for backup. - -services: - prod: - build: - context: . - args: - APP_VERSION: ${APP_VERSION:-unknown} - GIT_COMMIT: ${GIT_COMMIT:-unknown} - BUILD_TIME: ${BUILD_TIME:-unknown} - image: corescope:latest - container_name: corescope-prod - restart: unless-stopped - extra_hosts: - - "host.docker.internal:host-gateway" - ports: - - "${PROD_HTTP_PORT:-80}:${PROD_HTTP_PORT:-80}" - - "${PROD_HTTPS_PORT:-443}:${PROD_HTTPS_PORT:-443}" - - "${PROD_MQTT_PORT:-1883}:1883" - volumes: - - ./config.json:/app/config.json:ro - - ./caddy-config/Caddyfile:/etc/caddy/Caddyfile:ro - - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - - caddy-data:/data/caddy - environment: - - NODE_ENV=production - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] - interval: 30s - timeout: 5s - retries: 3 - - staging: - build: - context: . - args: - APP_VERSION: ${APP_VERSION:-unknown} - GIT_COMMIT: ${GIT_COMMIT:-unknown} - BUILD_TIME: ${BUILD_TIME:-unknown} - image: corescope:latest - container_name: corescope-staging - restart: unless-stopped - extra_hosts: - - "host.docker.internal:host-gateway" - ports: - - "${STAGING_HTTP_PORT:-81}:${STAGING_HTTP_PORT:-81}" - - "${STAGING_MQTT_PORT:-1884}:1883" - volumes: - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}/config.json:/app/config.json:ro - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}/Caddyfile:/etc/caddy/Caddyfile:ro - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - - caddy-data-staging:/data/caddy - environment: - - NODE_ENV=staging - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] - interval: 30s - timeout: 5s - retries: 3 - profiles: - - staging - - staging-go: - build: - context: . - dockerfile: Dockerfile - args: - APP_VERSION: ${APP_VERSION:-unknown} - GIT_COMMIT: ${GIT_COMMIT:-unknown} - BUILD_TIME: ${BUILD_TIME:-unknown} - image: corescope-go:latest - container_name: corescope-staging-go - restart: unless-stopped - extra_hosts: - - "host.docker.internal:host-gateway" - ports: - - "${STAGING_GO_HTTP_PORT:-82}:80" - - "${STAGING_GO_MQTT_PORT:-1885}:1883" - - "6060:6060" # pprof server - - "6061:6061" # pprof ingestor - volumes: - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}/config.json:/app/config.json:ro - - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - - caddy-data-staging-go:/data/caddy - environment: - - NODE_ENV=staging - - ENABLE_PPROF=true - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] - interval: 30s - timeout: 5s - retries: 3 - profiles: - - staging-go - -volumes: - # Named volumes for Caddy TLS certificates (not user data — managed by Caddy internally) +# All container config lives here. manage.sh is just a wrapper around docker compose. +# Override defaults via .env or environment variables. +# CRITICAL: All data mounts use bind mounts (~/path), NOT named volumes. +# This ensures the DB and theme are visible on the host filesystem for backup. + +services: + prod: + build: + context: . + args: + APP_VERSION: ${APP_VERSION:-unknown} + GIT_COMMIT: ${GIT_COMMIT:-unknown} + BUILD_TIME: ${BUILD_TIME:-unknown} + image: corescope:latest + container_name: corescope-prod + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "${PROD_HTTP_PORT:-80}:${PROD_HTTP_PORT:-80}" + - "${PROD_HTTPS_PORT:-443}:${PROD_HTTPS_PORT:-443}" + - "${PROD_MQTT_PORT:-1883}:1883" + volumes: + - ./config.json:/app/config.json:ro + - ./caddy-config/Caddyfile:/etc/caddy/Caddyfile:ro + - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data + - caddy-data:/data/caddy + environment: + - NODE_ENV=production + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + interval: 30s + timeout: 5s + retries: 3 + +volumes: + # Named volumes for Caddy TLS certificates (not user data — managed by Caddy internally) caddy-data: - caddy-data-staging: - caddy-data-staging-go: diff --git a/manage.sh b/manage.sh index e0bba8bd..59dc9ab8 100755 --- a/manage.sh +++ b/manage.sh @@ -18,6 +18,7 @@ STATE_FILE=".setup-state" # Resolved paths for prod/staging data (must match docker-compose.yml) PROD_DATA="${PROD_DATA_DIR:-$HOME/meshcore-data}" STAGING_DATA="${STAGING_DATA_DIR:-$HOME/meshcore-staging-data}" +STAGING_COMPOSE_FILE="docker-compose.staging.yml" # Build metadata — exported so docker compose build picks them up via args export APP_VERSION=$(node -p "require('./package.json').version" 2>/dev/null || echo "unknown") @@ -391,7 +392,7 @@ prepare_staging_db() { # Copy config.prod.json → config.staging.json with siteName change prepare_staging_config() { - local prod_config="$PROD_DATA/config.json" + local prod_config="./config.json" local staging_config="$STAGING_DATA/config.json" if [ ! -f "$prod_config" ]; then warn "No config.json found at ${prod_config} — staging may not start correctly." @@ -440,10 +441,11 @@ cmd_start() { prepare_staging_config info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." - info "Starting staging container (corescope-staging) on port ${STAGING_HTTP_PORT:-81}..." - docker compose --profile staging up -d + info "Starting staging container (corescope-staging-go) on port ${STAGING_GO_HTTP_PORT:-82}..." + docker compose up -d prod + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d staging-go log "Production started on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}/${PROD_MQTT_PORT:-1883}" - log "Staging started on port ${STAGING_HTTP_PORT:-81} (MQTT: ${STAGING_MQTT_PORT:-1884})" + log "Staging started on port ${STAGING_GO_HTTP_PORT:-82} (MQTT: ${STAGING_GO_MQTT_PORT:-1885})" else info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." docker compose up -d prod @@ -461,13 +463,16 @@ cmd_stop() { log "Production stopped." ;; staging) - info "Stopping staging container (corescope-staging)..." - docker compose --profile staging stop staging - log "Staging stopped." + info "Stopping staging container (corescope-staging-go)..." + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging rm -sf staging-go 2>/dev/null || true + docker rm -f corescope-staging-go meshcore-staging-go corescope-staging meshcore-staging 2>/dev/null || true + log "Staging stopped and cleaned up." ;; all) info "Stopping all containers..." - docker compose --profile staging --profile staging-go down + docker compose stop prod + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging rm -sf staging-go 2>/dev/null || true + docker rm -f corescope-staging-go meshcore-staging-go corescope-staging meshcore-staging 2>/dev/null || true log "All containers stopped." ;; *) @@ -486,13 +491,18 @@ cmd_restart() { log "Production restarted." ;; staging) - info "Restarting staging container (corescope-staging)..." - docker compose --profile staging up -d --force-recreate staging + info "Restarting staging container (corescope-staging-go)..." + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging rm -sf staging-go 2>/dev/null || true + docker rm -f corescope-staging-go 2>/dev/null || true + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d staging-go log "Staging restarted." ;; all) info "Restarting all containers..." - docker compose --profile staging up -d --force-recreate + docker compose up -d --force-recreate prod + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging rm -sf staging-go 2>/dev/null || true + docker rm -f corescope-staging-go 2>/dev/null || true + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d staging-go log "All containers restarted." ;; *) @@ -544,10 +554,10 @@ cmd_status() { echo "" # Staging - if container_running "corescope-staging"; then - show_container_status "corescope-staging" "Staging" + if container_running "corescope-staging-go"; then + show_container_status "corescope-staging-go" "Staging" else - info "Staging (corescope-staging): Not running (use --with-staging to start both)" + info "Staging (corescope-staging-go): Not running (use --with-staging to start both)" fi echo "" @@ -579,7 +589,7 @@ cmd_logs() { staging) if container_running "corescope-staging"; then info "Tailing staging logs..." - docker compose logs -f --tail="$LINES" staging + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging logs -f --tail="$LINES" staging-go else err "Staging container is not running." info "Start with: ./manage.sh start --with-staging" @@ -607,7 +617,7 @@ cmd_promote() { # Show what's currently running local staging_image staging_created prod_image prod_created - staging_image=$(docker inspect corescope-staging --format '{{.Config.Image}}' 2>/dev/null || echo "not running") + staging_image=$(docker inspect corescope-staging-go --format '{{.Config.Image}}' 2>/dev/null || echo "not running") staging_created=$(docker inspect corescope-staging --format '{{.Created}}' 2>/dev/null || echo "N/A") prod_image=$(docker inspect corescope-prod --format '{{.Config.Image}}' 2>/dev/null || echo "not running") prod_created=$(docker inspect corescope-prod --format '{{.Created}}' 2>/dev/null || echo "N/A") @@ -853,7 +863,8 @@ cmd_reset() { exit 0 fi - docker compose --profile staging --profile staging-go down --rmi local 2>/dev/null || true + docker compose down --rmi local 2>/dev/null || true + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging down --rmi local 2>/dev/null || true rm -f "$STATE_FILE" log "Reset complete. Run './manage.sh setup' to start over." @@ -874,7 +885,7 @@ cmd_help() { echo "" printf '%b\n' " ${BOLD}Run${NC}" echo " start Start production container" - echo " start --with-staging Start production + staging (copies prod DB + config)" + echo " start --with-staging Start production + staging-go (copies prod DB + config)" echo " stop [prod|staging|all] Stop specific or all containers (default: all)" echo " restart [prod|staging|all] Restart specific or all containers" echo " status Show health, stats, and service status" @@ -887,7 +898,7 @@ cmd_help() { echo " restore Restore from backup dir or .db file" echo " mqtt-test Check if MQTT data is flowing" echo "" - echo "All commands use docker compose with docker-compose.yml." + echo "Prod uses docker-compose.yml; staging uses ${STAGING_COMPOSE_FILE}." echo "" } From 900cbf6392079c3e79dd0dbdfac1955f14fc90f8 Mon Sep 17 00:00:00 2001 From: Kpa-clawbot <259247574+Kpa-clawbot@users.noreply.github.com> Date: Sun, 29 Mar 2026 14:06:37 -0700 Subject: [PATCH 2/2] fix: deploy uses manage.sh restart staging instead of raw compose Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/deploy.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cea04497..92c2bbbb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -261,24 +261,20 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - - name: Stop old staging - run: ./manage.sh stop staging - - - name: Start staging - run: | - docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d "$STAGING_SERVICE" + - name: Deploy staging + run: ./manage.sh restart staging - name: Healthcheck staging container run: | for i in $(seq 1 120); do - HEALTH=$(docker inspect "$STAGING_CONTAINER" --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") + HEALTH=$(docker inspect corescope-staging-go --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") if [ "$HEALTH" = "healthy" ]; then echo "Staging healthy after ${i}s" break fi if [ "$i" -eq 120 ]; then echo "Staging failed health check after 120s" - docker logs "$STAGING_CONTAINER" --tail 50 + docker logs corescope-staging-go --tail 50 exit 1 fi sleep 1