diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a52a740..dee2bb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,6 +393,8 @@ jobs: run: scripts/test-vacuum-maintenance.sh - name: Test immutable release rollback and schema compatibility gates run: scripts/test-replace-container.sh + - name: Test split infrastructure resolver dry run + run: scripts/test-infra-compose.sh secrets: name: Secret scan diff --git a/docs/runbook-backup-restore.md b/docs/runbook-backup-restore.md index a473b3a..df8cb1c 100644 --- a/docs/runbook-backup-restore.md +++ b/docs/runbook-backup-restore.md @@ -59,6 +59,12 @@ export MESHCORE_INFRA_PROJECT_NAME=meshcore-infra scripts/backup.sh ``` +`MESHCORE_INFRA_COMPOSE_FILE` can select a non-default external Compose file; +`MESHCORE_INFRA_PROJECT_DIR` can set its project directory explicitly. The +shared resolver checks that Compose project and requires one running container +for each exact `timescaledb`, `redis`, and `mosquitto` service. It does not +search container names across projects. + For the returned receipt and archive: ```bash diff --git a/docs/runbook-release-rollback.md b/docs/runbook-release-rollback.md index d411347..aeeba7c 100644 --- a/docs/runbook-release-rollback.md +++ b/docs/runbook-release-rollback.md @@ -45,6 +45,8 @@ export COSIGN_CERTIFICATE_OIDC_ISSUER='https://token.actions.githubusercontent.c export RESTORE_RECEIPT_PATH=/secure/receipts/latest.json export RESTORE_RECEIPT_VERIFY_KEY=/secure/receipts/verify.pem export RELEASE_RECEIPT_SIGNING_KEY=/secure/release/receipt-signing.pem +export MESHCORE_INFRA_DIR=/home/ben/ukmesh/meshcore-infra +export MESHCORE_INFRA_PROJECT_NAME=meshcore-infra scripts/replace-container.sh backend \ --image="$BACKEND_IMAGE" \ @@ -52,6 +54,13 @@ scripts/replace-container.sh backend \ --source-revision="$SOURCE_REVISION" ``` +If the infrastructure Compose file has another path, set +`MESHCORE_INFRA_COMPOSE_FILE` and, when needed, +`MESHCORE_INFRA_PROJECT_DIR`. The release script validates that external +project and resolves its single running `timescaledb` container before it +starts the migration job; the post-migration schema check uses that exact +container ID. + Repeat for each application service, passing its digest as `--image` and the same signed backend digest as `--backend-image`. The script verifies signatures and revision labels, runs migrations, starts the prior backend against the diff --git a/scripts/backup.sh b/scripts/backup.sh index 38ef029..cb0bec0 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -5,8 +5,18 @@ umask 077 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" project_dir="$(cd -- "${script_dir}/.." && pwd)" app_project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}" -infra_project_dir="${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}" +if [ -n "${MESHCORE_INFRA_COMPOSE_FILE:-}" ]; then + infra_compose_file="$MESHCORE_INFRA_COMPOSE_FILE" + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$infra_compose_file")}}" +else + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}}" + infra_compose_file="$infra_project_dir/docker-compose.yml" +fi infra_project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}" +export MESHCORE_INFRA_PROJECT_DIR="$infra_project_dir" +export MESHCORE_INFRA_COMPOSE_FILE="$infra_compose_file" +export MESHCORE_INFRA_PROJECT_NAME="$infra_project_name" +source "$script_dir/lib/infra-compose.sh" output_dir="${BACKUP_OUTPUT_DIR:?BACKUP_OUTPUT_DIR must name an encrypted backup target}" encryption_cert="${BACKUP_ENCRYPTION_CERT:?BACKUP_ENCRYPTION_CERT must name a public X.509 certificate}" signing_key="${BACKUP_RECEIPT_SIGNING_KEY:?BACKUP_RECEIPT_SIGNING_KEY must name the offline receipt signing key}" @@ -29,8 +39,8 @@ for path in "$encryption_cert" "$signing_key"; do exit 66 } done -test -f "$infra_project_dir/docker-compose.yml" || { - echo "meshcore-infra compose file is missing: $infra_project_dir/docker-compose.yml" >&2 +test -f "$infra_compose_file" || { + echo "infrastructure Compose file is missing: $infra_compose_file" >&2 exit 66 } openssl x509 -in "$encryption_cert" -noout >/dev/null @@ -66,30 +76,27 @@ if (( (8#$env_mode & 8#077) != 0 )); then echo "protected runtime configuration .env must not be group/world accessible" >&2 exit 66 fi +infra_env_file="$infra_project_dir/.env" +if [ ! -f "$infra_env_file" ] || [ -L "$infra_env_file" ]; then + echo "protected infrastructure configuration .env is missing or unsafe" >&2 + exit 66 +fi +infra_env_mode="$(stat -c '%a' "$infra_env_file")" +if (( (8#$infra_env_mode & 8#077) != 0 )); then + echo "protected infrastructure configuration .env must not be group/world accessible" >&2 + exit 66 +fi app_compose=( docker compose --project-directory "$project_dir" -f "$project_dir/docker-compose.yml" -f "$project_dir/docker-compose.live.yml" --project-name "$app_project_name" ) -infra_compose=( - docker compose --project-directory "$infra_project_dir" - -f "$infra_project_dir/docker-compose.yml" - --project-name "$infra_project_name" -) "${app_compose[@]}" config -q -"${infra_compose[@]}" config -q +meshcore_infra_compose config -q declare -A service_container_ids=() for service in timescaledb redis mosquitto; do - container_id="$("${infra_compose[@]}" ps -q "$service")" - test -n "$container_id" || { - echo "required service is not running: $service" >&2 - exit 69 - } - if [ "$(docker inspect "$container_id" --format '{{.State.Status}}')" != "running" ]; then - echo "required service is not running: $service" >&2 - exit 69 - fi + container_id="$(resolve_infra_container "$service")" service_container_ids["$service"]="$container_id" done timescaledb_container="${service_container_ids[timescaledb]}" @@ -190,8 +197,15 @@ tar -C "$project_dir" -czf "$payload_dir/configuration.tgz" \ logging \ backend/src/db/migrations backend/src/db/schema backend/src/db/owner-auth.sql \ scripts/backup.sh scripts/restore-drill.sh scripts/sync-latest.sh scripts/replace-container.sh \ + scripts/lib/infra-compose.sh scripts/unify-networks.sh scripts/relabel-packets-per-chunk.sh \ scripts/bootstrap-mosquitto.sh vacuum-compressed-chunks.sh -tar -C "$infra_project_dir" -czf "$payload_dir/infra-configuration.tgz" \ +infra_configuration_stage="$tmp_dir/infra-configuration" +mkdir -p "$infra_configuration_stage" +cp -- "$infra_project_dir/.env" "$infra_configuration_stage/.env" +cp -- "$infra_compose_file" "$infra_configuration_stage/docker-compose.yml" +cp -- "$infra_project_dir/README.md" "$infra_configuration_stage/README.md" +cp -- "$infra_project_dir/ROLLBACK.md" "$infra_configuration_stage/ROLLBACK.md" +tar -C "$infra_configuration_stage" -czf "$payload_dir/infra-configuration.tgz" \ .env docker-compose.yml README.md ROLLBACK.md completed_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" diff --git a/scripts/lib/infra-compose.sh b/scripts/lib/infra-compose.sh new file mode 100644 index 0000000..c76cf2d --- /dev/null +++ b/scripts/lib/infra-compose.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +meshcore_infra_compose() { + local project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}" + local project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-}}" + local compose_file="${MESHCORE_INFRA_COMPOSE_FILE:-}" + + if [ -z "$compose_file" ]; then + if [ -z "$project_dir" ]; then + echo 'MESHCORE_INFRA_PROJECT_DIR or MESHCORE_INFRA_COMPOSE_FILE is required' >&2 + return 64 + fi + compose_file="$project_dir/docker-compose.yml" + fi + if [ -z "$project_dir" ]; then + project_dir="$(dirname -- "$compose_file")" + fi + if [ ! -f "$compose_file" ]; then + echo "infrastructure Compose file is missing: $compose_file" >&2 + return 66 + fi + + docker compose \ + --project-directory "$project_dir" \ + --project-name "$project_name" \ + -f "$compose_file" \ + "$@" +} + +resolve_infra_container() { + local service="${1:-}" + case "$service" in + timescaledb|redis|mosquitto) ;; + *) + echo "unsupported infrastructure service: ${service:-}" >&2 + return 64 + ;; + esac + + local raw_ids + if ! raw_ids="$(meshcore_infra_compose ps -q "$service")"; then + echo "could not resolve infrastructure service: $service" >&2 + return 69 + fi + + local -a ids=() + local id + while IFS= read -r id; do + id="${id//$'\r'/}" + if [ -n "${id//[[:space:]]/}" ]; then + ids+=("$id") + fi + done <<<"$raw_ids" + if [ "${#ids[@]}" -ne 1 ]; then + echo "expected one infrastructure container for $service; found ${#ids[@]}" >&2 + return 69 + fi + + local state + if ! state="$(docker inspect --format '{{.State.Status}}' "${ids[0]}")"; then + echo "could not inspect infrastructure container for $service" >&2 + return 69 + fi + if [ "$state" != 'running' ]; then + echo "infrastructure container for $service is not running" >&2 + return 69 + fi + + printf '%s\n' "${ids[0]}" +} diff --git a/scripts/newuser.sh b/scripts/newuser.sh index 88aff81..7009172 100755 --- a/scripts/newuser.sh +++ b/scripts/newuser.sh @@ -3,11 +3,22 @@ set -Eeuo pipefail IFS=$'\n\t' +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" readonly REPO_DIR="${NEWUSER_REPO_DIR:-/home/ben/ukmesh/meshcore-analytics}" readonly ENV_FILE="${REPO_DIR}/.env" -readonly MOSQUITTO_CONTAINER="$(docker ps --format '{{.Names}}' | grep -E '^meshcore-infra-mosquitto-1$|mosquitto-1$' | head -1)" +if [ -n "${MESHCORE_INFRA_COMPOSE_FILE:-}" ]; then + infra_compose_file="$MESHCORE_INFRA_COMPOSE_FILE" + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$infra_compose_file")}}" +else + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$REPO_DIR")/meshcore-infra}}" + infra_compose_file="$infra_project_dir/docker-compose.yml" +fi +export MESHCORE_INFRA_PROJECT_DIR="$infra_project_dir" +export MESHCORE_INFRA_COMPOSE_FILE="$infra_compose_file" +source "$SCRIPT_DIR/lib/infra-compose.sh" +readonly MOSQUITTO_CONTAINER="$(resolve_infra_container mosquitto)" readonly BACKEND_CONTAINER='meshcore-analytics-backend-1' -readonly TIMESCALEDB_CONTAINER="$(docker ps --format '{{.Names}}' | grep -E '^meshcore-infra-timescaledb-1$|timescaledb-1$' | head -1)" +readonly TIMESCALEDB_CONTAINER="$(resolve_infra_container timescaledb)" readonly OWNER_DATABASE='meshcore_owner_auth' readonly POSTGRES_USER='meshcore' readonly BROKER_URL='wss://mqtt.ukmesh.com:443' diff --git a/scripts/package.json b/scripts/package.json index 2029351..d30aa00 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -6,7 +6,7 @@ "scripts": { "generate-key": "tsx generate-observer-key.ts", "register-observer": "tsx register-observer.ts", - "test": "bash test-generate-observer-key.sh && bash test-newuser.sh", + "test": "bash test-generate-observer-key.sh && bash test-infra-compose.sh && bash test-newuser.sh", "typecheck": "tsc --noEmit --target ES2022 --module NodeNext --moduleResolution NodeNext --strict --types node generate-observer-key.ts register-observer.ts" }, "dependencies": { diff --git a/scripts/relabel-packets-per-chunk.sh b/scripts/relabel-packets-per-chunk.sh index 3b13867..bcc9253 100755 --- a/scripts/relabel-packets-per-chunk.sh +++ b/scripts/relabel-packets-per-chunk.sh @@ -3,11 +3,25 @@ set -Eeuo pipefail cd "$(dirname "$0")/.." +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +project_dir="$(cd -- "$script_dir/.." && pwd)" +if [ -n "${MESHCORE_INFRA_COMPOSE_FILE:-}" ]; then + infra_compose_file="$MESHCORE_INFRA_COMPOSE_FILE" + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$infra_compose_file")}}" +else + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}}" + infra_compose_file="$infra_project_dir/docker-compose.yml" +fi +export MESHCORE_INFRA_PROJECT_DIR="$infra_project_dir" +export MESHCORE_INFRA_COMPOSE_FILE="$infra_compose_file" +source "$script_dir/lib/infra-compose.sh" + log() { printf '[%s] %s\n' "$(date -Is)" "$*" } -PSQL=(docker compose exec -T timescaledb psql -U "${POSTGRES_USER:-meshcore}" -d "${POSTGRES_DB:-meshcore}" -v ON_ERROR_STOP=1 -X) +timescaledb_container="$(resolve_infra_container timescaledb)" +PSQL=(docker exec -i "$timescaledb_container" psql -U "${POSTGRES_USER:-meshcore}" -d "${POSTGRES_DB:-meshcore}" -v ON_ERROR_STOP=1 -X) # `test` remains an isolated network. This maintenance operation only folds # historical production labels into the unified UKMesh scope. LEGACY_SQL="'teesside','northeast'" diff --git a/scripts/replace-container.sh b/scripts/replace-container.sh index 2325278..3a199e9 100755 --- a/scripts/replace-container.sh +++ b/scripts/replace-container.sh @@ -49,6 +49,16 @@ done script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" project_dir="$(cd -- "${script_dir}/.." && pwd)" project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}" +if [ -n "${MESHCORE_INFRA_COMPOSE_FILE:-}" ]; then + infra_compose_file="$MESHCORE_INFRA_COMPOSE_FILE" + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$infra_compose_file")}}" +else + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}}" + infra_compose_file="$infra_project_dir/docker-compose.yml" +fi +export MESHCORE_INFRA_PROJECT_DIR="$infra_project_dir" +export MESHCORE_INFRA_COMPOSE_FILE="$infra_compose_file" +source "$script_dir/lib/infra-compose.sh" release_dir="${RELEASE_STATUS_DIR:-/home/ben/meshcore-releases}" cosign_key="${COSIGN_PUBLIC_KEY:-}" cosign_identity_regexp="${COSIGN_CERTIFICATE_IDENTITY_REGEXP:-}" @@ -126,6 +136,8 @@ if ! docker compose --project-name "$project_name" config --services | grep -Fxq echo "unknown Compose service: $service" >&2 exit 65 fi +meshcore_infra_compose config -q +timescaledb_container="$(resolve_infra_container timescaledb)" if [ "$(git rev-parse HEAD)" != "$source_revision" ]; then echo "working checkout does not match the release source revision" >&2 exit 65 @@ -255,7 +267,7 @@ echo "Running the required migration job with the signed backend image..." BACKEND_IMAGE="$backend_image" \ docker compose --project-name "$project_name" run --rm db-migrate schema_version="$( - docker compose --project-name "$project_name" exec -T timescaledb \ + docker exec -i "$timescaledb_container" \ psql -U "${POSTGRES_USER:-meshcore}" -d "${POSTGRES_DB:-meshcore}" -Atc \ "SELECT COALESCE(MAX(((regexp_match(name, '^([0-9]+)_'))[1])::int), 0) FROM schema_migrations" )" diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index 838c8e5..474f337 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -53,6 +53,8 @@ test "$actual_archive_sha" = "$expected_archive_sha" || { exit 65 } +# This drill uses generated container names on a private network; all restore +# and smoke commands target those disposable containers directly. run_id="meshcore-restore-$$" network_name="${run_id}-network" postgres_name="${run_id}-postgres" diff --git a/scripts/test-infra-compose.sh b/scripts/test-infra-compose.sh new file mode 100755 index 0000000..c832ca9 --- /dev/null +++ b/scripts/test-infra-compose.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +test_root="$(mktemp -d)" +trap 'rm -rf -- "$test_root"' EXIT + +mkdir -p "$test_root/bin" "$test_root/external-infra" +cat >"$test_root/external-infra/compose.production.yml" <<'YAML' +services: + timescaledb: + image: timescale/timescaledb:fixture + redis: + image: redis:fixture + mosquitto: + image: eclipse-mosquitto:fixture +YAML +cat >"$test_root/bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$MOCK_DOCKER_LOG" + +command_name="${1:-}" +shift || true +case "$command_name" in + compose) + while [ "$#" -gt 0 ]; do + case "$1" in + --project-directory|--project-name|-f) + shift 2 + ;; + config) + exit 0 + ;; + ps) + shift + test "${1:-}" = '-q' + service="${2:-}" + if [ "${MOCK_PS_OUTPUT+x}" = x ]; then + printf '%s\n' "$MOCK_PS_OUTPUT" + else + printf 'container-%s\n' "$service" + fi + exit 0 + ;; + *) + echo "unexpected mock Compose command: $*" >&2 + exit 90 + ;; + esac + done + ;; + inspect) + if [ "${MOCK_STATE:-running}" = 'running' ]; then + printf 'running\n' + else + printf '%s\n' "$MOCK_STATE" + fi + ;; + *) + echo "unexpected mock Docker command: $command_name $*" >&2 + exit 91 + ;; +esac +EOF +chmod 0755 "$test_root/bin/docker" + +export PATH="$test_root/bin:$PATH" +export MOCK_DOCKER_LOG="$test_root/docker.log" +export MESHCORE_INFRA_PROJECT_DIR="$test_root/external-infra" +export MESHCORE_INFRA_COMPOSE_FILE="$test_root/external-infra/compose.production.yml" +export MESHCORE_INFRA_PROJECT_NAME='meshcore-infra-test' +source "$script_dir/lib/infra-compose.sh" + +meshcore_infra_compose config -q +for service in timescaledb redis mosquitto; do + resolved="$(resolve_infra_container "$service")" + test "$resolved" = "container-$service" + grep -Fq -- \ + "compose --project-directory $MESHCORE_INFRA_PROJECT_DIR --project-name $MESHCORE_INFRA_PROJECT_NAME -f $MESHCORE_INFRA_COMPOSE_FILE ps -q $service" \ + "$MOCK_DOCKER_LOG" +done + +unset MESHCORE_INFRA_PROJECT_DIR MESHCORE_INFRA_DIR +test "$(resolve_infra_container timescaledb)" = 'container-timescaledb' +grep -Fq -- \ + "compose --project-directory $(dirname -- "$MESHCORE_INFRA_COMPOSE_FILE") --project-name $MESHCORE_INFRA_PROJECT_NAME -f $MESHCORE_INFRA_COMPOSE_FILE ps -q timescaledb" \ + "$MOCK_DOCKER_LOG" + +if MOCK_PS_OUTPUT=$'container-one\ncontainer-two' resolve_infra_container timescaledb >/dev/null 2>&1; then + echo 'ambiguous service resolution unexpectedly succeeded' >&2 + exit 1 +fi +if MOCK_PS_OUTPUT='' resolve_infra_container redis >/dev/null 2>&1; then + echo 'missing service resolution unexpectedly succeeded' >&2 + exit 1 +fi +if MOCK_STATE=exited resolve_infra_container mosquitto >/dev/null 2>&1; then + echo 'stopped service resolution unexpectedly succeeded' >&2 + exit 1 +fi +if resolve_infra_container backend >/dev/null 2>&1; then + echo 'unsupported service resolution unexpectedly succeeded' >&2 + exit 1 +fi + +printf 'split infrastructure resolver dry-run tests passed\n' diff --git a/scripts/test-newuser.sh b/scripts/test-newuser.sh index fac0530..8b5e51f 100755 --- a/scripts/test-newuser.sh +++ b/scripts/test-newuser.sh @@ -9,7 +9,8 @@ readonly NODE_C='C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C test_root="$(mktemp -d)" trap 'rm -rf -- "$test_root"' EXIT -mkdir -p "$test_root/bin" "$test_root/repo" +mkdir -p "$test_root/bin" "$test_root/repo" "$test_root/infra" +touch "$test_root/infra/docker-compose.yml" printf 'OWNER_MQTT_USERNAME_MAP=alice=%s,bob=%s\n' "$NODE_A" "$NODE_C" \ >"$test_root/repo/.env" @@ -20,9 +21,6 @@ set -Eeuo pipefail printf 'ARGS %s\n' "$*" >>"$FAKE_DOCKER_LOG" case ${1:-} in - ps) - printf '%s\n' meshcore-infra-mosquitto-1 meshcore-infra-timescaledb-1 - ;; inspect) if [[ $* == *State.Running* ]]; then printf 'true\n' @@ -31,6 +29,12 @@ case ${1:-} in fi ;; compose) + case "$*" in + *'ps -q mosquitto') printf 'infra-mosquitto-id\n' ;; + *'ps -q timescaledb') printf 'infra-timescaledb-id\n' ;; + *' up -d --no-deps backend') ;; + *) printf 'unexpected docker compose invocation: %s\n' "$*" >&2; exit 1 ;; + esac ;; exec) sql='' @@ -60,6 +64,7 @@ export FAKE_DOCKER_LOG="$test_root/docker.log" output="$test_root/output.log" PATH="$test_root/bin:$PATH" \ NEWUSER_REPO_DIR="$test_root/repo" \ +MESHCORE_INFRA_PROJECT_DIR="$test_root/infra" \ TMPDIR="$test_root" \ bash "$SCRIPT_DIR/newuser.sh" --link alice "$NODE_B" >"$output" 2>&1 @@ -70,6 +75,10 @@ grep -Fq 'Password: unchanged (existing credential)' "$output" grep -Fq 'INSERT INTO owner_account_nodes' "$FAKE_DOCKER_LOG" grep -Fq 'compose -f docker-compose.yml -f docker-compose.live.yml up -d --no-deps backend' \ "$FAKE_DOCKER_LOG" +grep -Fq "compose --project-directory $test_root/infra --project-name meshcore-infra -f $test_root/infra/docker-compose.yml ps -q mosquitto" \ + "$FAKE_DOCKER_LOG" +grep -Fq "compose --project-directory $test_root/infra --project-name meshcore-infra -f $test_root/infra/docker-compose.yml ps -q timescaledb" \ + "$FAKE_DOCKER_LOG" if grep -Fq 'mosquitto_passwd -b' "$FAKE_DOCKER_LOG"; then printf 'link flow unexpectedly changed the existing credential\n' >&2 exit 1 diff --git a/scripts/test-replace-container.sh b/scripts/test-replace-container.sh index ac2af8a..1973c5d 100755 --- a/scripts/test-replace-container.sh +++ b/scripts/test-replace-container.sh @@ -61,46 +61,64 @@ command_name="${1:-}" shift || true case "$command_name" in compose) - if [ "${1:-}" = "--project-name" ]; then - shift 2 - fi - case "${1:-}" in - config) - if [ "${2:-}" = "--services" ]; then - printf '%s\n' backend db-migrate timescaledb - elif [ "${2:-}" != "-q" ]; then - printf 'services: {}\n' - fi - ;; - ps) - printf 'current-backend\n' - ;; - run) - test "${*: -1}" = "db-migrate" - ;; - exec) - if [[ " $* " == *" timescaledb "* ]]; then - printf '30\n' - elif [[ "$*" == *"/readyz"* ]]; then - printf '{"status":"ready"}\n' - elif [[ "$*" == *"/metrics"* ]]; then - printf 'meshcore_process_start_time_seconds 1\n' - else - printf 'unexpected docker compose exec invocation: %s\n' "$*" >&2 - exit 91 - fi - ;; - up) - if [ "${MOCK_FAIL_DESIRED_DEPLOY:-false}" = "true" ] \ - && [ "${BACKEND_IMAGE:-}" = "$MOCK_DESIRED_IMAGE" ]; then - exit 1 - fi - ;; - *) - printf 'unexpected docker compose invocation: %s\n' "$*" >&2 - exit 92 - ;; - esac + project_name='' + while [ "$#" -gt 0 ]; do + case "$1" in + --project-name) + project_name="$2" + shift 2 + ;; + --project-directory|-f) + shift 2 + ;; + config) + if [ "${2:-}" = "--services" ]; then + printf '%s\n' backend db-migrate + elif [ "${2:-}" != "-q" ]; then + printf 'services: {}\n' + fi + exit 0 + ;; + ps) + shift + test "${1:-}" = '-q' + service="${2:-}" + if [ "$project_name" = 'meshcore-infra' ]; then + test "$service" = 'timescaledb' + printf 'infra-timescaledb\n' + else + printf 'current-backend\n' + fi + exit 0 + ;; + run) + test "${*: -1}" = "db-migrate" + exit 0 + ;; + exec) + if [[ "$*" == *"/readyz"* ]]; then + printf '{"status":"ready"}\n' + elif [[ "$*" == *"/metrics"* ]]; then + printf 'meshcore_process_start_time_seconds 1\n' + else + printf 'unexpected docker compose exec invocation: %s\n' "$*" >&2 + exit 91 + fi + exit 0 + ;; + up) + if [ "${MOCK_FAIL_DESIRED_DEPLOY:-false}" = "true" ] \ + && [ "${BACKEND_IMAGE:-}" = "$MOCK_DESIRED_IMAGE" ]; then + exit 1 + fi + exit 0 + ;; + *) + printf 'unexpected docker compose invocation: %s\n' "$*" >&2 + exit 92 + ;; + esac + done ;; pull) ;; @@ -124,6 +142,9 @@ case "$command_name" in *'.State.Running'*) printf 'true\n' ;; + *'.State.Status'*) + printf 'running\n' + ;; *'.State.Health'*) printf 'healthy\n' ;; @@ -137,6 +158,10 @@ case "$command_name" in printf 'compat-container-id\n' ;; exec) + if [[ "$*" == *"infra-timescaledb"*"psql"* ]]; then + printf '30\n' + exit 0 + fi if [ "${MOCK_COMPAT_READY:-true}" != "true" ]; then exit 1 fi @@ -178,12 +203,15 @@ run_case() { trap 'rm -rf -- "$test_root"' RETURN mkdir -p \ - "${test_root}/project/scripts" \ + "${test_root}/project/scripts/lib" \ + "${test_root}/infra" \ "${test_root}/fake-bin" \ "${test_root}/releases" cp "$replace_script" "${test_root}/project/scripts/replace-container.sh" + cp "${script_dir}/lib/infra-compose.sh" "${test_root}/project/scripts/lib/infra-compose.sh" chmod 0755 "${test_root}/project/scripts/replace-container.sh" printf 'services: {}\n' >"${test_root}/project/docker-compose.yml" + printf 'services: {}\n' >"${test_root}/infra/docker-compose.yml" local now now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" @@ -228,6 +256,7 @@ run_case() { RESTORE_RECEIPT_PATH="${test_root}/restore-receipt.json" \ RESTORE_RECEIPT_VERIFY_KEY="${test_root}/receipt-verify.pem" \ RELEASE_STATUS_DIR="${test_root}/releases" \ + MESHCORE_INFRA_PROJECT_DIR="${test_root}/infra" \ COMPATIBILITY_TIMEOUT_SECONDS=1 \ "${test_root}/project/scripts/replace-container.sh" \ backend \ @@ -252,6 +281,12 @@ run_case() { test "$(jq -r '.status' "$release_status")" = "$expected_status" test "$(jq -r '.schema_version' "$release_status")" = "30" test "$(jq -r '.prior_image' "$release_status")" = "$prior_image" + local resolver_line migration_line + resolver_line="$(grep -nF 'ps -q timescaledb' "${test_root}/docker.log" | cut -d: -f1)" + migration_line="$(grep -nF 'compose --project-name meshcore-analytics run --rm db-migrate' "${test_root}/docker.log" | cut -d: -f1)" + test "$resolver_line" -lt "$migration_line" + grep -Fq 'exec -i infra-timescaledb psql -U meshcore -d meshcore -Atc' \ + "${test_root}/docker.log" local up_count up_count="$( diff --git a/scripts/unify-networks.sh b/scripts/unify-networks.sh index 5792a10..069061d 100755 --- a/scripts/unify-networks.sh +++ b/scripts/unify-networks.sh @@ -1,10 +1,24 @@ #!/usr/bin/env bash set -Eeuo pipefail -cd "$(dirname "$0")/.." +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +project_dir="$(cd -- "$script_dir/.." && pwd)" +cd "$project_dir" + +if [ -n "${MESHCORE_INFRA_COMPOSE_FILE:-}" ]; then + infra_compose_file="$MESHCORE_INFRA_COMPOSE_FILE" + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$infra_compose_file")}}" +else + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}}" + infra_compose_file="$infra_project_dir/docker-compose.yml" +fi +export MESHCORE_INFRA_PROJECT_DIR="$infra_project_dir" +export MESHCORE_INFRA_COMPOSE_FILE="$infra_compose_file" +source "$script_dir/lib/infra-compose.sh" MODE="${1:-audit}" -PSQL=(docker compose exec -T timescaledb psql -U "${POSTGRES_USER:-meshcore}" -d "${POSTGRES_DB:-meshcore}" -v ON_ERROR_STOP=1 -X) +timescaledb_container="$(resolve_infra_container timescaledb)" +PSQL=(docker exec -i "$timescaledb_container" psql -U "${POSTGRES_USER:-meshcore}" -d "${POSTGRES_DB:-meshcore}" -v ON_ERROR_STOP=1 -X) LEGACY_SQL="'teesside','northeast'" RUN_ID="${NETWORK_UNIFICATION_RUN_ID:-network-unification-$(date -u +%Y%m%dT%H%M%SZ)}" diff --git a/vacuum-compressed-chunks.sh b/vacuum-compressed-chunks.sh index abec917..459d5b5 100755 --- a/vacuum-compressed-chunks.sh +++ b/vacuum-compressed-chunks.sh @@ -59,18 +59,26 @@ if [ -n "$chunk" ]; then fi cd "$repo_dir" -# Resolve the compose project that owns the timescaledb service. The test harness -# drives the script with COMPOSE_FILE/COMPOSE_PROJECT_NAME set (its own fixture); -# in the two-project production layout the database lives in the separate -# meshcore-infra project (mirrors scripts/backup.sh). -if [ -z "${COMPOSE_FILE:-}" ]; then - infra_dir="${MESHCORE_INFRA_DIR:-${repo_dir}/../meshcore-infra}" - compose_base=(docker compose --project-directory "$infra_dir" -f "$infra_dir/docker-compose.yml" --project-name "${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}") +if [ -n "${MESHCORE_INFRA_COMPOSE_FILE:-}" ]; then + infra_compose_file="$MESHCORE_INFRA_COMPOSE_FILE" + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-$(dirname -- "$infra_compose_file")}}" + infra_project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}" +elif [ -n "${COMPOSE_FILE:-}" ]; then + infra_compose_file="$COMPOSE_FILE" + infra_project_dir="$(dirname -- "$infra_compose_file")" + infra_project_name="${COMPOSE_PROJECT_NAME:-meshcore-infra}" else - compose_base=(docker compose) + infra_project_dir="${MESHCORE_INFRA_PROJECT_DIR:-${MESHCORE_INFRA_DIR:-${repo_dir}/../meshcore-infra}}" + infra_compose_file="$infra_project_dir/docker-compose.yml" + infra_project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}" fi -"${compose_base[@]}" config -q -psql_command=("${compose_base[@]}" exec -T timescaledb psql -X -v ON_ERROR_STOP=1 -U "${POSTGRES_USER:-meshcore}" -d "$database") +export MESHCORE_INFRA_PROJECT_DIR="$infra_project_dir" +export MESHCORE_INFRA_COMPOSE_FILE="$infra_compose_file" +export MESHCORE_INFRA_PROJECT_NAME="$infra_project_name" +source "$repo_dir/scripts/lib/infra-compose.sh" +meshcore_infra_compose config -q +timescaledb_container="$(resolve_infra_container timescaledb)" +psql_command=(docker exec -i "$timescaledb_container" psql -X -v ON_ERROR_STOP=1 -U "${POSTGRES_USER:-meshcore}" -d "$database") database_exists="$("${psql_command[@]}" -Atc \ "SELECT 1 FROM pg_database WHERE datname = current_database() AND NOT datistemplate")" @@ -138,7 +146,7 @@ case "$action" in ;; esac available_bytes="$( - "${compose_base[@]}" exec -T timescaledb \ + docker exec "$timescaledb_container" \ df -PB1 /var/lib/postgresql/data | awk 'NR==2 {print $4}' )"