From d11336da648c46dbd477b65440ec57e25e6fb718 Mon Sep 17 00:00:00 2001 From: gadgethd <111318106+gadgethd@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:31:15 +0000 Subject: [PATCH] fix(backup): restore split-stack recovery workflow --- .gitignore | 2 + backup-receipts/latest.json | 29 ++++----- backup-receipts/latest.json.sig | Bin 384 -> 384 bytes backup-receipts/verify.pem | 18 +++--- docs/runbook-backup-restore.md | 55 +++++++++++++++- scripts/backup.sh | 68 ++++++++++++++------ scripts/restore-drill.sh | 61 ++++++++++++------ scripts/sync-latest.sh | 108 ++++++++++++++++++++++++++++++++ 8 files changed, 277 insertions(+), 64 deletions(-) create mode 100755 scripts/sync-latest.sh diff --git a/.gitignore b/.gitignore index b814908..8da2091 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,8 @@ audit.md NEW_PATHING_SYSTEM.md local-vs-github-pr-plan.md /.env.bak-* +/backups/ +/backup-receipts/owner-grants-*.json terrain-tiles/ diff --git a/backup-receipts/latest.json b/backup-receipts/latest.json index 6a0ac05..d589130 100644 --- a/backup-receipts/latest.json +++ b/backup-receipts/latest.json @@ -1,30 +1,31 @@ { "format": "meshcore-restore-receipt-v1", - "receipt_id": "restore-20260729T230204Z-20260729T202342Z-b9b2ecdbfaa9", - "backup_id": "backup-20260729T202342Z-b9b2ecdbfaa9", - "backup_completed_at": "2026-07-29T20:33:10.000Z", - "restore_started_at": "2026-07-29T22:45:33.000Z", - "restore_verified_at": "2026-07-29T23:02:04.000Z", - "restore_duration_seconds": 991, - "recovery_point_age_seconds": 8934, - "archive_sha256": "75489f067f0698b350e7804ef48ae7e89e4e5ad4287cf2cd21da62860862d713", - "source_revision": "b9b2ecdbfaa9688a79a62ccee38d6b406f14deaa", - "schema_version": 30, + "receipt_id": "restore-20260813T032656Z-20260813T031400Z-2c1dd39b7173", + "backup_id": "backup-20260813T031400Z-2c1dd39b7173", + "backup_completed_at": "2026-08-13T03:18:09.000Z", + "restore_started_at": "2026-08-13T03:20:02.000Z", + "restore_verified_at": "2026-08-13T03:26:56.000Z", + "restore_duration_seconds": 414, + "recovery_point_age_seconds": 527, + "archive_sha256": "6466077779be65d3f843843b3fbf57cd33abad671391c5744e93d3a0fe1911a7", + "source_revision": "2c1dd39b7173e2cb97164a6e9dbe9e1f600b6aaa", + "schema_version": 51, "datasets": [ "analytics", "owner_auth", "mosquitto", "redis", - "configuration" + "configuration", + "infra_configuration" ], "evidence": { "integrity": { "packet_sample_rows": 1000, - "packet_sample_md5": "ccb1301a8f4adc96b01640ed99127c87", + "packet_sample_md5": "d975a0fd34eee068df4eabf80f7d52ae", "node_sample_rows": 1000, - "node_sample_md5": "21e4151aebbb4a7275f6a2f7ede22cf6" + "node_sample_md5": "5d1a8fea57b96209ad404739fbaa6628" }, - "owner_lookup_count": 44 + "owner_lookup_count": 47 }, "checks": { "migrations": "passed", diff --git a/backup-receipts/latest.json.sig b/backup-receipts/latest.json.sig index 1ad33ee9b74bbe6ee10a01f40871792f208696ea..b26b6742ee9cf0e645cc89fdcce237830e28ba51 100644 GIT binary patch literal 384 zcmV-`0e}7$tY0LYwD;n#dZBq`){80q$4)NoeoQU1S9c57^#%-Gth#(apT^Sn=Bn}U!NCS9r_s8EwF|N&K7nIP_68L=sv{cKuMLnt1G%Frm?8 zw@MH>@G@(Qxh3E^Bt*+bgpjfCxR{fg4kAWZ6SkZse7ertUkYHkzYr2*I literal 384 zcmV-`0e}7oWjkCj68QMgOp;I1E*~XfSaPpFaqKed%)6sGf@Fy*gGzux+0GQSCA0xJ z5!2VE6jYL3eW)6__R0$--t}q80E<8E&>(2(TG6(Wy@ZU?D_WzuWKs;PY#V^A>5=I` zBxMPqVR*YeXO#lUdr3tLH(YDzDini^kOIHg@o`Eu;fwxVjCcz82s8vpieTK<8S__7 zvnHY>=I$uIdt&M)W677Ac0XFa`O0pH%0BcXGNWO0p}F`->4RE3Eg}^_3K#6e%nAF7 zqbU(&m@XE-yp;)XoThy@G-B?7F7QvY-x)Lf0nb`U>di?-@|XTts-2*^bjp$=fxC~D z;HWI8F3VP3(9Gy3j!ANqC_G{PNXI&M%=ySIx<~Zeo+MC5H#0HsoHbuP~DnF-l=5lWqZrxuUvtV`&&s?Sp0_AgTrY6 diff --git a/backup-receipts/verify.pem b/backup-receipts/verify.pem index a4c8195..752900b 100644 --- a/backup-receipts/verify.pem +++ b/backup-receipts/verify.pem @@ -1,11 +1,11 @@ -----BEGIN PUBLIC KEY----- -MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7vYpsp5q11oyAwNG7ApJ -xUCJ2xU2TGb2JjmJdpU2BVvD4FHvfCHyMJW7uUnKsYhKM5v0sEktTfuhF4e1K6I2 -pn0OwJ+3oYgEuprQTbSfr4SkmafL6SCY12kc7RrZeLAyhiqqrCrfp6Ilr7rSRZSa -Cb+7MXYh4TESzXCfz9ZlAQgIKwohuuhRRDOahwaDqh2zKpVCPgnWAvYxWaaDLS8o -wqvNN0Ephpg+cK44RLRnZX/jz/emCB12XkdNkZpvsaoo8pdYa2ztq0wIH0t4oWa+ -GEHPfLafnzSVwmCaVFjJR8bIb9G78veAjmzpjfdDbcaE7jK/V6tMgE92ed9naeJX -mS3aLyWvfJbEtKFZ/yPFACvHh1we/y4s6RkpD9iU4VYaLYE3fEDQp50sWbsyXf9d -0aPGjlQfhgOQDyaauzXX64YE52Pcd9FlzBr6/PlBN7IqOsNNibNRtk4z1RcfwS4E -6UyDlgFK1bLqM5lVQd5pkEnln180CaheIDLzuUn1ZRpVAgMBAAE= +MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAzmrnF4/1t9kblReJT0wg +NPVZFYDgXqZuNhoebpdo3zF4cve00xg6owk/EMgPFtvLUfqJPAma/ZDeE/Co6PJQ +XNdEvu35oIS5huB2mf7qKa0lO6lVEtZHSRFIEBNP6HfLg4y+5i0MDZBkN5zFms7p +CAAl/V7FFv3F2/EkK1CYRsGT42r519P9X+apFaa0hHguiOHDjgcgWQK24HrJGVyd +a3ZXmATPSdQPSEQzRXM0U12ejpUsr0MXjbYLkUr/lalpNqfOcfqoWVGAky4uExfO +N4xuBrviTwn9t6AXKOlIjR+EBKv7Wc/Agt+vcbq3Bj/zKf7odSIJWoyl/74iio/z +bEozCWO2GiDC+7oV+hKnlIsQ97HcN5+9wHkR3D80tUUUKfO4BvVUHZzhtq/LhB1V +PBFkQ0R6WSlwrVQgbT5KKaEukRyJZ34ZlFIbhsKSi6qjHsH2XCMcGi4z4XE50AqT +CtuAuNr4Gmytajnz+yYUQpNAKCM3+M1y+pa9lgMdLxxZAgMBAAE= -----END PUBLIC KEY----- diff --git a/docs/runbook-backup-restore.md b/docs/runbook-backup-restore.md index 905938b..a473b3a 100644 --- a/docs/runbook-backup-restore.md +++ b/docs/runbook-backup-restore.md @@ -5,7 +5,10 @@ state, Mosquitto persistence and credentials, and the configuration needed to reconstruct the release. The configuration payload includes the protected `.env` recovery secrets, Compose/Docker build definitions, edge policies, monitoring rules, database schema/migrations, and the recovery/deployment tools -themselves. `.env` must be a regular file with no group/world permissions. The +themselves. It contains both app Compose overlays and the separately managed +`meshcore-infra` Compose file/configuration; data-plane reads are addressed only +through the explicit `meshcore-infra` project. `.env` files must be regular +files with no group/world permissions. The set is encrypted before leaving its access-controlled temporary storage and has a detached signed receipt. The initial objective is RPO 24 hours and RTO four hours. @@ -51,6 +54,8 @@ The script rejects a target on the application filesystem unless export BACKUP_OUTPUT_DIR=/mnt/offsite/meshcore export BACKUP_ENCRYPTION_CERT=/secure/meshcore-backup-keys/encrypt.pem export BACKUP_RECEIPT_SIGNING_KEY=/secure/meshcore-backup-keys/receipt-signing.pem +export MESHCORE_INFRA_DIR=/home/ben/ukmesh/meshcore-infra +export MESHCORE_INFRA_PROJECT_NAME=meshcore-infra scripts/backup.sh ``` @@ -69,6 +74,48 @@ Copy the receipt, signature, and public verification key alongside the encrypted archive. Alert if any required dataset lacks a verified receipt or the newest receipt is older than 24 hours. +If backup creation stages under `/home/ben/ukmesh/backups`, publish the newest +complete set to a separately mounted and attested filesystem with atomic names +and an independently checked checksum manifest: + +```bash +export BACKUP_SOURCE_DIR=/home/ben/ukmesh/backups +export BACKUP_SYNC_TARGET_DIR=/mnt/offsite/meshcore +export BACKUP_RECEIPT_VERIFY_KEY=/secure/meshcore-backup-keys/receipt-verify.pem +scripts/sync-latest.sh +(cd /mnt/offsite/meshcore && sha256sum --check latest.sha256) +``` + +`BACKUP_SYNC_ALLOW_LOCAL_TEST=true` is drill-only; it does not satisfy the +off-host pre-reset gate. + +## Owner authorization reset gate + +Before the backup, export every active grant (including database-only and +legacy/null-method rows) without changing operator configuration: + +```bash +install -d -m 0700 /home/ben/ukmesh/backups +cd backend && npm run build +node --env-file=../.env dist/tools/inventoryOwnerAuthorization.js \ + --export=/home/ben/ukmesh/backups/owner-grants-YYYY-MM-DDTHHMMSSZ.json +``` + +The export is secret, mode 0600, checksum-bearing, and must never be committed. +The legacy disposition is explicit preservation pending Ben's promote/revoke +decision. Restore `owner-auth.dump`, mount the export read-only, and set: + +```dotenv +OWNER_AUTH_INVENTORY_HOST_DIR=/home/ben/ukmesh/backups +OWNER_AUTH_INVENTORY_BASELINE_PATH=/owner-auth-inventory/owner-grants-YYYY-MM-DDTHHMMSSZ.json +``` + +Before public admission, run `owner-auth:inventory -- --baseline= +--require-complete`. `/readyz` also fails closed until active account/grant and +per-method counts, configured/desired/rendered/applied generations, last error, +and ACL readback exactly match the pre-reset baseline. Do not rewrite +`OWNER_MQTT_USERNAME_MAP` as part of this preservation step. + ## Isolated restore drill The drill creates a private Docker network and disposable PostgreSQL, Redis, @@ -102,8 +149,10 @@ Do not restore directly over a running production volume. 4. Create fresh production volumes and restore using the same ordered dataset procedure as the drill. 5. Run migrations from the signed target backend image. -6. Boot the backend with MQTT ingest disabled; verify readiness, privacy-scoped - stats, owner lookup, queue invariants, and Mosquitto authentication. +6. Restore `owner-auth.dump` and the owner inventory baseline before starting + the reconciler. Boot with MQTT ingest disabled; require the CLI baseline + validation and `/readyz`, then verify privacy-scoped stats, owner lookup, + queue invariants, and Mosquitto authentication. 7. Reopen public reads, then workers, then MQTT ingest. 8. Preserve the failed volumes read-only until incident review completes. diff --git a/scripts/backup.sh b/scripts/backup.sh index 505ebf6..38ef029 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -4,7 +4,9 @@ umask 077 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" project_dir="$(cd -- "${script_dir}/.." && pwd)" -project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}" +app_project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}" +infra_project_dir="${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}" +infra_project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}" output_dir="${BACKUP_OUTPUT_DIR:?BACKUP_OUTPUT_DIR must name an encrypted backup target}" encryption_cert="${BACKUP_ENCRYPTION_CERT:?BACKUP_ENCRYPTION_CERT must name a public X.509 certificate}" signing_key="${BACKUP_RECEIPT_SIGNING_KEY:?BACKUP_RECEIPT_SIGNING_KEY must name the offline receipt signing key}" @@ -27,6 +29,10 @@ for path in "$encryption_cert" "$signing_key"; do exit 66 } done +test -f "$infra_project_dir/docker-compose.yml" || { + echo "meshcore-infra compose file is missing: $infra_project_dir/docker-compose.yml" >&2 + exit 66 +} openssl x509 -in "$encryption_cert" -noout >/dev/null openssl pkey -in "$signing_key" -noout >/dev/null @@ -60,9 +66,22 @@ if (( (8#$env_mode & 8#077) != 0 )); then echo "protected runtime configuration .env must not be group/world accessible" >&2 exit 66 fi -docker compose --project-name "$project_name" config -q +app_compose=( + docker compose --project-directory "$project_dir" + -f "$project_dir/docker-compose.yml" + -f "$project_dir/docker-compose.live.yml" + --project-name "$app_project_name" +) +infra_compose=( + docker compose --project-directory "$infra_project_dir" + -f "$infra_project_dir/docker-compose.yml" + --project-name "$infra_project_name" +) +"${app_compose[@]}" config -q +"${infra_compose[@]}" config -q +declare -A service_container_ids=() for service in timescaledb redis mosquitto; do - container_id="$(docker compose --project-name "$project_name" ps -q "$service")" + container_id="$("${infra_compose[@]}" ps -q "$service")" test -n "$container_id" || { echo "required service is not running: $service" >&2 exit 69 @@ -71,7 +90,11 @@ for service in timescaledb redis mosquitto; do echo "required service is not running: $service" >&2 exit 69 fi + service_container_ids["$service"]="$container_id" done +timescaledb_container="${service_container_ids[timescaledb]}" +redis_container="${service_container_ids[redis]}" +mosquitto_container="${service_container_ids[mosquitto]}" started_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" backup_id="backup-$(date -u +%Y%m%dT%H%M%SZ)-$(git rev-parse --short=12 HEAD)" @@ -104,33 +127,34 @@ cleanup() { trap cleanup EXIT echo "Creating consistent PostgreSQL custom-format dumps..." -docker compose --project-name "$project_name" exec -T timescaledb \ +docker exec -i "$timescaledb_container" \ pg_dump -U "$db_user" -d "$analytics_db" \ --format=custom --compress=6 --no-owner --no-acl \ >"$payload_dir/analytics.dump" -docker compose --project-name "$project_name" exec -T timescaledb \ +docker exec -i "$timescaledb_container" \ pg_dump -U "$db_user" -d "$owner_db" \ --format=custom --compress=6 --no-owner --no-acl \ >"$payload_dir/owner-auth.dump" schema_version="$( - docker compose --project-name "$project_name" exec -T timescaledb \ + docker exec -i "$timescaledb_container" \ psql -U "$db_user" -d "$analytics_db" -Atc \ "SELECT COALESCE(MAX(((regexp_match(name, '^([0-9]+)_'))[1])::int), 0) FROM schema_migrations" )" schema_version="${schema_version//$'\r'/}" echo "Checkpointing Redis durable queues and state..." -docker compose --project-name "$project_name" exec -T redis sh -c \ +docker exec -i "$redis_container" sh -c \ 'redis-cli -a "$REDIS_PASSWORD" --no-auth-warning BGSAVE >/dev/null || true' redis_ready="false" for _ in $(seq 1 120); do persistence="$( - docker compose --project-name "$project_name" exec -T redis sh -c \ + docker exec -i "$redis_container" sh -c \ 'redis-cli -a "$REDIS_PASSWORD" --no-auth-warning INFO persistence' )" - if printf '%s' "$persistence" | tr -d '\r' | grep -q '^rdb_bgsave_in_progress:0$' \ - && printf '%s' "$persistence" | tr -d '\r' | grep -q '^rdb_last_bgsave_status:ok$'; then + persistence="${persistence//$'\r'/}" + if grep -q '^rdb_bgsave_in_progress:0$' <<<"$persistence" \ + && grep -q '^rdb_last_bgsave_status:ok$' <<<"$persistence"; then redis_ready="true" break fi @@ -140,27 +164,35 @@ test "$redis_ready" = "true" || { echo "Redis persistence checkpoint did not complete successfully" >&2 exit 75 } -docker compose --project-name "$project_name" exec -T redis \ +docker exec -i "$redis_container" \ tar -C /data -czf - . >"$payload_dir/redis-state.tgz" echo "Flushing and capturing Mosquitto credentials, ACL, and persistence..." -docker compose --project-name "$project_name" exec -T mosquitto sh -c \ +docker exec -i "$mosquitto_container" sh -c \ 'kill -USR1 1; sleep 1; tar -C /mosquitto/data -czf - .' \ >"$payload_dir/mosquitto-state.tgz" -tar -C "$project_dir" -czf "$payload_dir/mosquitto-config.tgz" \ +mosquitto_config_stage="$tmp_dir/mosquitto-configuration" +mkdir -p "$mosquitto_config_stage/mosquitto" +docker exec -i "$mosquitto_container" \ + tar -C /mosquitto/config -cf - mosquitto.conf acl passwd \ + | tar -C "$mosquitto_config_stage/mosquitto" -xf - +tar -C "$mosquitto_config_stage" -czf "$payload_dir/mosquitto-config.tgz" \ mosquitto/mosquitto.conf mosquitto/acl mosquitto/passwd tar -C "$project_dir" -czf "$payload_dir/configuration.tgz" \ .env .env.example .trivyignore.yaml \ - docker-compose.yml \ + docker-compose.yml docker-compose.live.yml docker-compose.ci.yml \ + docker-compose.infra-client.yml compose.phase4.yml \ Dockerfile Dockerfile.app Dockerfile.backend Dockerfile.website \ Dockerfile.mesh-health-check Dockerfile.mosquitto-reloader \ - viewshed-worker/Dockerfile ml-path-learner/Dockerfile \ + viewshed-worker/Dockerfile \ nginx.app.conf nginx.website.conf nginx.security-headers.conf \ anubis/botPolicy.yaml docker/mosquitto-reloader.py \ logging \ backend/src/db/migrations backend/src/db/schema backend/src/db/owner-auth.sql \ - scripts/backup.sh scripts/restore-drill.sh scripts/replace-container.sh \ + scripts/backup.sh scripts/restore-drill.sh scripts/sync-latest.sh scripts/replace-container.sh \ scripts/bootstrap-mosquitto.sh vacuum-compressed-chunks.sh +tar -C "$infra_project_dir" -czf "$payload_dir/infra-configuration.tgz" \ + .env docker-compose.yml README.md ROLLBACK.md completed_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" manifest_path="$payload_dir/manifest.json" @@ -195,7 +227,7 @@ jq -n \ started_at: $started_at, completed_at: $completed_at, schema_version: $schema_version, - datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration"], + datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"], files: $files }' >"$manifest_path" @@ -252,7 +284,7 @@ jq -n \ archive: $archive, archive_sha256: $archive_sha256, archive_bytes: $archive_bytes, - datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration"], + datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"], encryption: "CMS-AES-256-CBC-CHUNKED-v1", status: "complete" }' >"$receipt_partial" diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh index a517881..838c8e5 100755 --- a/scripts/restore-drill.sh +++ b/scripts/restore-drill.sh @@ -62,6 +62,7 @@ backend_name="${run_id}-backend" postgres_volume="${run_id}-postgres-data" redis_volume="${run_id}-redis-data" mosquitto_volume="${run_id}-mosquitto-data" +mosquitto_config_volume="${run_id}-mosquitto-config" application_image="${run_id}-application" tmp_dir="$(mktemp -d)" payload_dir="$tmp_dir/payload" @@ -98,7 +99,7 @@ cleanup() { >/dev/null 2>&1 || true docker network rm "$network_name" >/dev/null 2>&1 || true docker volume rm \ - "$postgres_volume" "$redis_volume" "$mosquitto_volume" \ + "$postgres_volume" "$redis_volume" "$mosquitto_volume" "$mosquitto_config_volume" \ >/dev/null 2>&1 || true docker image rm "$application_image" >/dev/null 2>&1 || true rm -rf "$tmp_dir" @@ -193,20 +194,31 @@ while IFS= read -r payload_name; do done < <(jq -r '.files | keys[]' "$manifest_path") for required_payload in \ analytics.dump owner-auth.dump redis-state.tgz mosquitto-state.tgz \ - mosquitto-config.tgz configuration.tgz; do + mosquitto-config.tgz configuration.tgz infra-configuration.tgz; do jq -e --arg name "$required_payload" '.files[$name] != null' \ "$manifest_path" >/dev/null || { echo "missing backup payload: $required_payload" >&2 exit 65 } done +infra_configuration_members="$(tar -tzf "$payload_dir/infra-configuration.tgz")" +for required_infra_configuration in \ + .env docker-compose.yml README.md ROLLBACK.md; do + grep -Fx "$required_infra_configuration" \ + <<<"$infra_configuration_members" >/dev/null || { + echo "missing protected infra recovery configuration: $required_infra_configuration" >&2 + exit 65 + } +done configuration_members="$(tar -tzf "$payload_dir/configuration.tgz")" for required_configuration in \ - .env docker-compose.yml Dockerfile.backend Dockerfile.mosquitto-reloader \ + .env docker-compose.yml docker-compose.live.yml docker-compose.ci.yml \ + docker-compose.infra-client.yml compose.phase4.yml \ + Dockerfile.backend Dockerfile.mosquitto-reloader \ anubis/botPolicy.yaml logging/prometheus.yml logging/alertmanager.yml \ - backend/src/db/schema/base.sql scripts/backup.sh scripts/restore-drill.sh \ + backend/src/db/schema/base.sql scripts/backup.sh scripts/restore-drill.sh scripts/sync-latest.sh \ scripts/replace-container.sh; do - printf '%s\n' "$configuration_members" | grep -Fxq "$required_configuration" || { + grep -Fx "$required_configuration" <<<"$configuration_members" >/dev/null || { echo "missing protected recovery configuration: $required_configuration" >&2 exit 65 } @@ -214,8 +226,8 @@ done mosquitto_configuration_members="$(tar -tzf "$payload_dir/mosquitto-config.tgz")" for required_mosquitto_configuration in \ mosquitto/mosquitto.conf mosquitto/acl mosquitto/passwd; do - printf '%s\n' "$mosquitto_configuration_members" \ - | grep -Fxq "$required_mosquitto_configuration" || { + grep -Fx "$required_mosquitto_configuration" \ + <<<"$mosquitto_configuration_members" >/dev/null || { echo "missing Mosquitto recovery configuration: $required_mosquitto_configuration" >&2 exit 65 } @@ -225,6 +237,7 @@ docker network create "$network_name" >/dev/null docker volume create "$postgres_volume" >/dev/null docker volume create "$redis_volume" >/dev/null docker volume create "$mosquitto_volume" >/dev/null +docker volume create "$mosquitto_config_volume" >/dev/null docker run -d \ --name "$postgres_name" \ @@ -237,13 +250,15 @@ docker run -d \ "$timescale_image" \ postgres -c autovacuum=off >/dev/null for _ in $(seq 1 240); do - if docker logs "$postgres_name" 2>&1 | grep -q 'PostgreSQL init process complete' && + postgres_logs="$(docker logs "$postgres_name" 2>&1)" + if grep -q 'PostgreSQL init process complete' <<<"$postgres_logs" && docker exec "$postgres_name" pg_isready -U meshcore -d meshcore >/dev/null 2>&1; then break fi sleep 0.5 done -docker logs "$postgres_name" 2>&1 | grep -q 'PostgreSQL init process complete' +postgres_logs="$(docker logs "$postgres_name" 2>&1)" +grep -q 'PostgreSQL init process complete' <<<"$postgres_logs" docker exec "$postgres_name" pg_isready -U meshcore -d meshcore >/dev/null docker exec "$postgres_name" createdb -U meshcore meshcore_owner_auth docker exec -i "$postgres_name" pg_restore \ @@ -255,11 +270,11 @@ docker exec -i "$postgres_name" pg_restore \ echo "PostgreSQL datasets restored." echo "Restoring Redis durable state..." -docker run --rm \ +docker run --rm -i \ -v "$redis_volume:/restore" \ - -v "$payload_dir:/payload:ro" \ "$busybox_image" \ - sh -c 'tar -C /restore -xzf /payload/redis-state.tgz' >/dev/null + sh -c 'tar -C /restore -xzf -' \ + <"$payload_dir/redis-state.tgz" >/dev/null docker run -d \ --name "$redis_name" \ --network "$network_name" \ @@ -273,8 +288,9 @@ docker run -d \ --maxmemory-policy noeviction >/dev/null redis_ready="false" for _ in $(seq 1 60); do - if docker exec "$redis_name" redis-cli -a "$restore_password" --no-auth-warning ping \ - 2>/dev/null | grep -q PONG; then + redis_ping="$(docker exec "$redis_name" redis-cli -a "$restore_password" --no-auth-warning ping \ + 2>/dev/null || true)" + if grep -Fxq PONG <<<"$redis_ping"; then redis_ready="true" break fi @@ -302,17 +318,22 @@ done # uid. The parent remains 0700 on the host, while this signed, short-lived # child directory is traversable only through the container's read-only bind. chmod 0755 "$tmp_dir/mosquitto" -docker run --rm \ +tar -C "$tmp_dir/mosquitto" -cf - mosquitto.conf acl passwd \ + | docker run --rm -i \ + -v "$mosquitto_config_volume:/restore" \ + "$busybox_image" \ + sh -c 'tar -C /restore -xf -' >/dev/null +docker run --rm -i \ -v "$mosquitto_volume:/restore" \ - -v "$payload_dir:/payload:ro" \ "$busybox_image" \ - sh -c 'tar -C /restore -xzf /payload/mosquitto-state.tgz; chown -R 1883:1883 /restore' \ + sh -c 'tar -C /restore -xzf -; chown -R 1883:1883 /restore' \ + <"$payload_dir/mosquitto-state.tgz" \ >/dev/null docker run -d \ --name "$mosquitto_name" \ --network "$network_name" \ --network-alias mosquitto \ - -v "$tmp_dir/mosquitto:/mosquitto/config:ro" \ + -v "$mosquitto_config_volume:/mosquitto/config:ro" \ -v "$mosquitto_volume:/mosquitto/data" \ "$mosquitto_image" >/dev/null mosquitto_ready="false" @@ -356,7 +377,7 @@ docker run -d \ --cap-drop ALL \ --security-opt no-new-privileges:true \ --tmpfs /tmp:rw,noexec,nosuid,size=64m \ - -v "$tmp_dir/mosquitto/acl:/mosquitto/config/acl:ro" \ + -v "$mosquitto_config_volume:/mosquitto/config:ro" \ -e "DATABASE_URL=$database_url" \ -e "OWNER_DATABASE_URL=$owner_database_url" \ -e "REDIS_URL=$redis_url" \ @@ -464,7 +485,7 @@ jq -n \ archive_sha256: $archive_sha256, source_revision: $source_revision, schema_version: $schema_version, - datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration"], + datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"], evidence: { integrity: $integrity, owner_lookup_count: $owner_lookup_count diff --git a/scripts/sync-latest.sh b/scripts/sync-latest.sh new file mode 100755 index 0000000..ba47dae --- /dev/null +++ b/scripts/sync-latest.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +source_dir="${BACKUP_SOURCE_DIR:-/home/ben/ukmesh/backups}" +target_dir="${BACKUP_SYNC_TARGET_DIR:?BACKUP_SYNC_TARGET_DIR must name the mounted off-host publication target}" +verify_key="${BACKUP_RECEIPT_VERIFY_KEY:?BACKUP_RECEIPT_VERIFY_KEY is required}" +allow_local="${BACKUP_SYNC_ALLOW_LOCAL_TEST:-false}" + +for command in openssl jq sha256sum findmnt find sort sed awk sync; do + command -v "$command" >/dev/null || { + echo "required command is unavailable: $command" >&2 + exit 69 + } +done +test -d "$source_dir" || { echo "backup source is missing: $source_dir" >&2; exit 66; } +test -r "$verify_key" || { echo "receipt verify key is unreadable" >&2; exit 66; } +mkdir -p "$target_dir" +target_dir="$(cd "$target_dir" && pwd)" +test -f "$target_dir/.meshcore-offsite-target" || { + echo "sync target is not attested (.meshcore-offsite-target missing)" >&2 + exit 65 +} +if [ "$allow_local" != "true" ] \ + && [ "$(findmnt -T "$source_dir" -n -o SOURCE)" = "$(findmnt -T "$target_dir" -n -o SOURCE)" ]; then + echo "sync target shares the backup source filesystem" >&2 + exit 65 +fi + +receipt_path="$( + find "$source_dir" -maxdepth 1 -type f -name 'backup-*.receipt.json' -printf '%T@ %p\n' \ + | sort -nr \ + | sed -n '1 { s/^[^ ]* //; p; }' +)" +test -n "$receipt_path" && test -f "$receipt_path" || { + echo "no completed backup receipt found" >&2 + exit 66 +} +signature_path="${receipt_path}.sig" +test -f "$signature_path" || { echo "backup receipt signature is missing" >&2; exit 66; } +openssl dgst -sha256 -verify "$verify_key" \ + -signature "$signature_path" "$receipt_path" >/dev/null +jq -e '.format == "meshcore-backup-receipt-v1" and .status == "complete"' \ + "$receipt_path" >/dev/null +archive_name="$(jq -er '.archive' "$receipt_path")" +case "$archive_name" in + */*|'') echo "backup receipt archive name is unsafe" >&2; exit 65 ;; +esac +archive_path="$source_dir/$archive_name" +test -f "$archive_path" || { echo "backup archive is missing" >&2; exit 66; } +expected_archive_sha="$(jq -er '.archive_sha256' "$receipt_path")" +actual_archive_sha="$(sha256sum "$archive_path" | awk '{print $1}')" +test "$actual_archive_sha" = "$expected_archive_sha" || { + echo "backup archive checksum does not match receipt" >&2 + exit 65 +} + +publication_id="$(jq -er '.backup_id' "$receipt_path")" +case "$publication_id" in + backup-[0-9A-Za-z._-]*) ;; + *) echo "backup id is unsafe" >&2; exit 65 ;; +esac +partial_dir="$target_dir/.${publication_id}.partial" +test ! -e "$partial_dir" || { echo "partial publication already exists" >&2; exit 73; } +mkdir "$partial_dir" +cleanup() { + if [ -d "$partial_dir" ]; then + find "$partial_dir" -maxdepth 1 -type f -delete + rmdir "$partial_dir" + fi +} +trap cleanup EXIT + +cp -- "$archive_path" "$partial_dir/$archive_name" +cp -- "$receipt_path" "$partial_dir/latest.receipt.json" +cp -- "$signature_path" "$partial_dir/latest.receipt.json.sig" +test "$(sha256sum "$partial_dir/$archive_name" | awk '{print $1}')" = "$expected_archive_sha" +openssl dgst -sha256 -verify "$verify_key" \ + -signature "$partial_dir/latest.receipt.json.sig" \ + "$partial_dir/latest.receipt.json" >/dev/null +( + cd "$partial_dir" + sha256sum "$archive_name" latest.receipt.json latest.receipt.json.sig \ + >latest.sha256 + sha256sum --check latest.sha256 >/dev/null +) +sync "$partial_dir/$archive_name" "$partial_dir/latest.receipt.json" \ + "$partial_dir/latest.receipt.json.sig" "$partial_dir/latest.sha256" + +mv "$partial_dir/$archive_name" "$target_dir/$archive_name" +mv "$partial_dir/latest.receipt.json" "$target_dir/latest.receipt.json" +mv "$partial_dir/latest.receipt.json.sig" "$target_dir/latest.receipt.json.sig" +mv "$partial_dir/latest.sha256" "$target_dir/latest.sha256" +rmdir "$partial_dir" +sync "$target_dir/$archive_name" "$target_dir/latest.receipt.json" \ + "$target_dir/latest.receipt.json.sig" "$target_dir/latest.sha256" +trap - EXIT + +( + cd "$target_dir" + sha256sum --check latest.sha256 >/dev/null +) +openssl dgst -sha256 -verify "$verify_key" \ + -signature "$target_dir/latest.receipt.json.sig" \ + "$target_dir/latest.receipt.json" >/dev/null +jq -n --arg backup_id "$publication_id" --arg target "$target_dir" \ + --arg archive "$archive_name" \ + '{backup_id: $backup_id, target: $target, archive: $archive, status: "verified"}'