fix(backup): restore split-stack recovery workflow

.gitignore

@@ -56,6 +56,8 @@ audit.md
 NEW_PATHING_SYSTEM.md
 local-vs-github-pr-plan.md
 /.env.bak-*
+/backups/
+/backup-receipts/owner-grants-*.json
 
 terrain-tiles/
 

backup-receipts/latest.json

@@ -1,30 +1,31 @@
 {
   "format": "meshcore-restore-receipt-v1",
-  "receipt_id": "restore-20260729T230204Z-20260729T202342Z-b9b2ecdbfaa9",
-  "backup_id": "backup-20260729T202342Z-b9b2ecdbfaa9",
-  "backup_completed_at": "2026-07-29T20:33:10.000Z",
-  "restore_started_at": "2026-07-29T22:45:33.000Z",
-  "restore_verified_at": "2026-07-29T23:02:04.000Z",
-  "restore_duration_seconds": 991,
-  "recovery_point_age_seconds": 8934,
-  "archive_sha256": "75489f067f0698b350e7804ef48ae7e89e4e5ad4287cf2cd21da62860862d713",
-  "source_revision": "b9b2ecdbfaa9688a79a62ccee38d6b406f14deaa",
-  "schema_version": 30,
+  "receipt_id": "restore-20260813T032656Z-20260813T031400Z-2c1dd39b7173",
+  "backup_id": "backup-20260813T031400Z-2c1dd39b7173",
+  "backup_completed_at": "2026-08-13T03:18:09.000Z",
+  "restore_started_at": "2026-08-13T03:20:02.000Z",
+  "restore_verified_at": "2026-08-13T03:26:56.000Z",
+  "restore_duration_seconds": 414,
+  "recovery_point_age_seconds": 527,
+  "archive_sha256": "6466077779be65d3f843843b3fbf57cd33abad671391c5744e93d3a0fe1911a7",
+  "source_revision": "2c1dd39b7173e2cb97164a6e9dbe9e1f600b6aaa",
+  "schema_version": 51,
   "datasets": [
     "analytics",
     "owner_auth",
     "mosquitto",
     "redis",
-    "configuration"
+    "configuration",
+    "infra_configuration"
   ],
   "evidence": {
     "integrity": {
       "packet_sample_rows": 1000,
-      "packet_sample_md5": "ccb1301a8f4adc96b01640ed99127c87",
+      "packet_sample_md5": "d975a0fd34eee068df4eabf80f7d52ae",
       "node_sample_rows": 1000,
-      "node_sample_md5": "21e4151aebbb4a7275f6a2f7ede22cf6"
+      "node_sample_md5": "5d1a8fea57b96209ad404739fbaa6628"
     },
-    "owner_lookup_count": 44
+    "owner_lookup_count": 47
   },
   "checks": {
     "migrations": "passed",

backup-receipts/verify.pem

@@ -1,11 +1,11 @@
 -----BEGIN PUBLIC KEY-----
-MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7vYpsp5q11oyAwNG7ApJ
-xUCJ2xU2TGb2JjmJdpU2BVvD4FHvfCHyMJW7uUnKsYhKM5v0sEktTfuhF4e1K6I2
-pn0OwJ+3oYgEuprQTbSfr4SkmafL6SCY12kc7RrZeLAyhiqqrCrfp6Ilr7rSRZSa
-Cb+7MXYh4TESzXCfz9ZlAQgIKwohuuhRRDOahwaDqh2zKpVCPgnWAvYxWaaDLS8o
-wqvNN0Ephpg+cK44RLRnZX/jz/emCB12XkdNkZpvsaoo8pdYa2ztq0wIH0t4oWa+
-GEHPfLafnzSVwmCaVFjJR8bIb9G78veAjmzpjfdDbcaE7jK/V6tMgE92ed9naeJX
-mS3aLyWvfJbEtKFZ/yPFACvHh1we/y4s6RkpD9iU4VYaLYE3fEDQp50sWbsyXf9d
-0aPGjlQfhgOQDyaauzXX64YE52Pcd9FlzBr6/PlBN7IqOsNNibNRtk4z1RcfwS4E
-6UyDlgFK1bLqM5lVQd5pkEnln180CaheIDLzuUn1ZRpVAgMBAAE=
+MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAzmrnF4/1t9kblReJT0wg
+NPVZFYDgXqZuNhoebpdo3zF4cve00xg6owk/EMgPFtvLUfqJPAma/ZDeE/Co6PJQ
+XNdEvu35oIS5huB2mf7qKa0lO6lVEtZHSRFIEBNP6HfLg4y+5i0MDZBkN5zFms7p
+CAAl/V7FFv3F2/EkK1CYRsGT42r519P9X+apFaa0hHguiOHDjgcgWQK24HrJGVyd
+a3ZXmATPSdQPSEQzRXM0U12ejpUsr0MXjbYLkUr/lalpNqfOcfqoWVGAky4uExfO
+N4xuBrviTwn9t6AXKOlIjR+EBKv7Wc/Agt+vcbq3Bj/zKf7odSIJWoyl/74iio/z
+bEozCWO2GiDC+7oV+hKnlIsQ97HcN5+9wHkR3D80tUUUKfO4BvVUHZzhtq/LhB1V
+PBFkQ0R6WSlwrVQgbT5KKaEukRyJZ34ZlFIbhsKSi6qjHsH2XCMcGi4z4XE50AqT
+CtuAuNr4Gmytajnz+yYUQpNAKCM3+M1y+pa9lgMdLxxZAgMBAAE=
 -----END PUBLIC KEY-----

docs/runbook-backup-restore.md

@@ -5,7 +5,10 @@ state, Mosquitto persistence and credentials, and the configuration needed to
 reconstruct the release. The configuration payload includes the protected
 `.env` recovery secrets, Compose/Docker build definitions, edge policies,
 monitoring rules, database schema/migrations, and the recovery/deployment tools
-themselves. `.env` must be a regular file with no group/world permissions. The
+themselves. It contains both app Compose overlays and the separately managed
+`meshcore-infra` Compose file/configuration; data-plane reads are addressed only
+through the explicit `meshcore-infra` project. `.env` files must be regular
+files with no group/world permissions. The
 set is encrypted before leaving its access-controlled temporary storage and has
 a detached signed receipt. The initial objective is RPO 24 hours and RTO four
 hours.
@@ -51,6 +54,8 @@ The script rejects a target on the application filesystem unless
 export BACKUP_OUTPUT_DIR=/mnt/offsite/meshcore
 export BACKUP_ENCRYPTION_CERT=/secure/meshcore-backup-keys/encrypt.pem
 export BACKUP_RECEIPT_SIGNING_KEY=/secure/meshcore-backup-keys/receipt-signing.pem
+export MESHCORE_INFRA_DIR=/home/ben/ukmesh/meshcore-infra
+export MESHCORE_INFRA_PROJECT_NAME=meshcore-infra
 scripts/backup.sh
 ```
 
@@ -69,6 +74,48 @@ Copy the receipt, signature, and public verification key alongside the
 encrypted archive. Alert if any required dataset lacks a verified receipt or
 the newest receipt is older than 24 hours.
 
+If backup creation stages under `/home/ben/ukmesh/backups`, publish the newest
+complete set to a separately mounted and attested filesystem with atomic names
+and an independently checked checksum manifest:
+
+```bash
+export BACKUP_SOURCE_DIR=/home/ben/ukmesh/backups
+export BACKUP_SYNC_TARGET_DIR=/mnt/offsite/meshcore
+export BACKUP_RECEIPT_VERIFY_KEY=/secure/meshcore-backup-keys/receipt-verify.pem
+scripts/sync-latest.sh
+(cd /mnt/offsite/meshcore && sha256sum --check latest.sha256)
+```
+
+`BACKUP_SYNC_ALLOW_LOCAL_TEST=true` is drill-only; it does not satisfy the
+off-host pre-reset gate.
+
+## Owner authorization reset gate
+
+Before the backup, export every active grant (including database-only and
+legacy/null-method rows) without changing operator configuration:
+
+```bash
+install -d -m 0700 /home/ben/ukmesh/backups
+cd backend && npm run build
+node --env-file=../.env dist/tools/inventoryOwnerAuthorization.js \
+  --export=/home/ben/ukmesh/backups/owner-grants-YYYY-MM-DDTHHMMSSZ.json
+```
+
+The export is secret, mode 0600, checksum-bearing, and must never be committed.
+The legacy disposition is explicit preservation pending Ben's promote/revoke
+decision. Restore `owner-auth.dump`, mount the export read-only, and set:
+
+```dotenv
+OWNER_AUTH_INVENTORY_HOST_DIR=/home/ben/ukmesh/backups
+OWNER_AUTH_INVENTORY_BASELINE_PATH=/owner-auth-inventory/owner-grants-YYYY-MM-DDTHHMMSSZ.json
+```
+
+Before public admission, run `owner-auth:inventory -- --baseline=<host-path>
+--require-complete`. `/readyz` also fails closed until active account/grant and
+per-method counts, configured/desired/rendered/applied generations, last error,
+and ACL readback exactly match the pre-reset baseline. Do not rewrite
+`OWNER_MQTT_USERNAME_MAP` as part of this preservation step.
+
 ## Isolated restore drill
 
 The drill creates a private Docker network and disposable PostgreSQL, Redis,
@@ -102,8 +149,10 @@ Do not restore directly over a running production volume.
 4. Create fresh production volumes and restore using the same ordered dataset
    procedure as the drill.
 5. Run migrations from the signed target backend image.
-6. Boot the backend with MQTT ingest disabled; verify readiness, privacy-scoped
-   stats, owner lookup, queue invariants, and Mosquitto authentication.
+6. Restore `owner-auth.dump` and the owner inventory baseline before starting
+   the reconciler. Boot with MQTT ingest disabled; require the CLI baseline
+   validation and `/readyz`, then verify privacy-scoped stats, owner lookup,
+   queue invariants, and Mosquitto authentication.
 7. Reopen public reads, then workers, then MQTT ingest.
 8. Preserve the failed volumes read-only until incident review completes.
 

scripts/backup.sh

@@ -4,7 +4,9 @@ umask 077
 
 script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 project_dir="$(cd -- "${script_dir}/.." && pwd)"
-project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}"
+app_project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}"
+infra_project_dir="${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}"
+infra_project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}"
 output_dir="${BACKUP_OUTPUT_DIR:?BACKUP_OUTPUT_DIR must name an encrypted backup target}"
 encryption_cert="${BACKUP_ENCRYPTION_CERT:?BACKUP_ENCRYPTION_CERT must name a public X.509 certificate}"
 signing_key="${BACKUP_RECEIPT_SIGNING_KEY:?BACKUP_RECEIPT_SIGNING_KEY must name the offline receipt signing key}"
@@ -27,6 +29,10 @@ for path in "$encryption_cert" "$signing_key"; do
     exit 66
   }
 done
+test -f "$infra_project_dir/docker-compose.yml" || {
+  echo "meshcore-infra compose file is missing: $infra_project_dir/docker-compose.yml" >&2
+  exit 66
+}
 openssl x509 -in "$encryption_cert" -noout >/dev/null
 openssl pkey -in "$signing_key" -noout >/dev/null
 
@@ -60,9 +66,22 @@ if (( (8#$env_mode & 8#077) != 0 )); then
   echo "protected runtime configuration .env must not be group/world accessible" >&2
   exit 66
 fi
-docker compose --project-name "$project_name" config -q
+app_compose=(
+  docker compose --project-directory "$project_dir"
+  -f "$project_dir/docker-compose.yml"
+  -f "$project_dir/docker-compose.live.yml"
+  --project-name "$app_project_name"
+)
+infra_compose=(
+  docker compose --project-directory "$infra_project_dir"
+  -f "$infra_project_dir/docker-compose.yml"
+  --project-name "$infra_project_name"
+)
+"${app_compose[@]}" config -q
+"${infra_compose[@]}" config -q
+declare -A service_container_ids=()
 for service in timescaledb redis mosquitto; do
-  container_id="$(docker compose --project-name "$project_name" ps -q "$service")"
+  container_id="$("${infra_compose[@]}" ps -q "$service")"
   test -n "$container_id" || {
     echo "required service is not running: $service" >&2
     exit 69
@@ -71,7 +90,11 @@ for service in timescaledb redis mosquitto; do
     echo "required service is not running: $service" >&2
     exit 69
   fi
+  service_container_ids["$service"]="$container_id"
 done
+timescaledb_container="${service_container_ids[timescaledb]}"
+redis_container="${service_container_ids[redis]}"
+mosquitto_container="${service_container_ids[mosquitto]}"
 
 started_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)"
 backup_id="backup-$(date -u +%Y%m%dT%H%M%SZ)-$(git rev-parse --short=12 HEAD)"
@@ -104,33 +127,34 @@ cleanup() {
 trap cleanup EXIT
 
 echo "Creating consistent PostgreSQL custom-format dumps..."
-docker compose --project-name "$project_name" exec -T timescaledb \
+docker exec -i "$timescaledb_container" \
   pg_dump -U "$db_user" -d "$analytics_db" \
   --format=custom --compress=6 --no-owner --no-acl \
   >"$payload_dir/analytics.dump"
-docker compose --project-name "$project_name" exec -T timescaledb \
+docker exec -i "$timescaledb_container" \
   pg_dump -U "$db_user" -d "$owner_db" \
   --format=custom --compress=6 --no-owner --no-acl \
   >"$payload_dir/owner-auth.dump"
 
 schema_version="$(
-  docker compose --project-name "$project_name" exec -T timescaledb \
+  docker exec -i "$timescaledb_container" \
     psql -U "$db_user" -d "$analytics_db" -Atc \
     "SELECT COALESCE(MAX(((regexp_match(name, '^([0-9]+)_'))[1])::int), 0) FROM schema_migrations"
 )"
 schema_version="${schema_version//$'\r'/}"
 
 echo "Checkpointing Redis durable queues and state..."
-docker compose --project-name "$project_name" exec -T redis sh -c \
+docker exec -i "$redis_container" sh -c \
   'redis-cli -a "$REDIS_PASSWORD" --no-auth-warning BGSAVE >/dev/null || true'
 redis_ready="false"
 for _ in $(seq 1 120); do
   persistence="$(
-    docker compose --project-name "$project_name" exec -T redis sh -c \
+    docker exec -i "$redis_container" sh -c \
       'redis-cli -a "$REDIS_PASSWORD" --no-auth-warning INFO persistence'
   )"
-  if printf '%s' "$persistence" | tr -d '\r' | grep -q '^rdb_bgsave_in_progress:0$' \
-    && printf '%s' "$persistence" | tr -d '\r' | grep -q '^rdb_last_bgsave_status:ok$'; then
+  persistence="${persistence//$'\r'/}"
+  if grep -q '^rdb_bgsave_in_progress:0$' <<<"$persistence" \
+    && grep -q '^rdb_last_bgsave_status:ok$' <<<"$persistence"; then
     redis_ready="true"
     break
   fi
@@ -140,27 +164,35 @@ test "$redis_ready" = "true" || {
   echo "Redis persistence checkpoint did not complete successfully" >&2
   exit 75
 }
-docker compose --project-name "$project_name" exec -T redis \
+docker exec -i "$redis_container" \
   tar -C /data -czf - . >"$payload_dir/redis-state.tgz"
 
 echo "Flushing and capturing Mosquitto credentials, ACL, and persistence..."
-docker compose --project-name "$project_name" exec -T mosquitto sh -c \
+docker exec -i "$mosquitto_container" sh -c \
   'kill -USR1 1; sleep 1; tar -C /mosquitto/data -czf - .' \
   >"$payload_dir/mosquitto-state.tgz"
-tar -C "$project_dir" -czf "$payload_dir/mosquitto-config.tgz" \
+mosquitto_config_stage="$tmp_dir/mosquitto-configuration"
+mkdir -p "$mosquitto_config_stage/mosquitto"
+docker exec -i "$mosquitto_container" \
+  tar -C /mosquitto/config -cf - mosquitto.conf acl passwd \
+  | tar -C "$mosquitto_config_stage/mosquitto" -xf -
+tar -C "$mosquitto_config_stage" -czf "$payload_dir/mosquitto-config.tgz" \
   mosquitto/mosquitto.conf mosquitto/acl mosquitto/passwd
 tar -C "$project_dir" -czf "$payload_dir/configuration.tgz" \
   .env .env.example .trivyignore.yaml \
-  docker-compose.yml \
+  docker-compose.yml docker-compose.live.yml docker-compose.ci.yml \
+  docker-compose.infra-client.yml compose.phase4.yml \
   Dockerfile Dockerfile.app Dockerfile.backend Dockerfile.website \
   Dockerfile.mesh-health-check Dockerfile.mosquitto-reloader \
-  viewshed-worker/Dockerfile ml-path-learner/Dockerfile \
+  viewshed-worker/Dockerfile \
   nginx.app.conf nginx.website.conf nginx.security-headers.conf \
   anubis/botPolicy.yaml docker/mosquitto-reloader.py \
   logging \
   backend/src/db/migrations backend/src/db/schema backend/src/db/owner-auth.sql \
-  scripts/backup.sh scripts/restore-drill.sh scripts/replace-container.sh \
+  scripts/backup.sh scripts/restore-drill.sh scripts/sync-latest.sh scripts/replace-container.sh \
   scripts/bootstrap-mosquitto.sh vacuum-compressed-chunks.sh
+tar -C "$infra_project_dir" -czf "$payload_dir/infra-configuration.tgz" \
+  .env docker-compose.yml README.md ROLLBACK.md
 
 completed_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)"
 manifest_path="$payload_dir/manifest.json"
@@ -195,7 +227,7 @@ jq -n \
     started_at: $started_at,
     completed_at: $completed_at,
     schema_version: $schema_version,
-    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration"],
+    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"],
     files: $files
   }' >"$manifest_path"
 
@@ -252,7 +284,7 @@ jq -n \
     archive: $archive,
     archive_sha256: $archive_sha256,
     archive_bytes: $archive_bytes,
-    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration"],
+    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"],
     encryption: "CMS-AES-256-CBC-CHUNKED-v1",
     status: "complete"
   }' >"$receipt_partial"

scripts/restore-drill.sh

@@ -62,6 +62,7 @@ backend_name="${run_id}-backend"
 postgres_volume="${run_id}-postgres-data"
 redis_volume="${run_id}-redis-data"
 mosquitto_volume="${run_id}-mosquitto-data"
+mosquitto_config_volume="${run_id}-mosquitto-config"
 application_image="${run_id}-application"
 tmp_dir="$(mktemp -d)"
 payload_dir="$tmp_dir/payload"
@@ -98,7 +99,7 @@ cleanup() {
     >/dev/null 2>&1 || true
   docker network rm "$network_name" >/dev/null 2>&1 || true
   docker volume rm \
-    "$postgres_volume" "$redis_volume" "$mosquitto_volume" \
+    "$postgres_volume" "$redis_volume" "$mosquitto_volume" "$mosquitto_config_volume" \
     >/dev/null 2>&1 || true
   docker image rm "$application_image" >/dev/null 2>&1 || true
   rm -rf "$tmp_dir"
@@ -193,20 +194,31 @@ while IFS= read -r payload_name; do
 done < <(jq -r '.files | keys[]' "$manifest_path")
 for required_payload in \
   analytics.dump owner-auth.dump redis-state.tgz mosquitto-state.tgz \
-  mosquitto-config.tgz configuration.tgz; do
+  mosquitto-config.tgz configuration.tgz infra-configuration.tgz; do
   jq -e --arg name "$required_payload" '.files[$name] != null' \
     "$manifest_path" >/dev/null || {
       echo "missing backup payload: $required_payload" >&2
       exit 65
     }
 done
+infra_configuration_members="$(tar -tzf "$payload_dir/infra-configuration.tgz")"
+for required_infra_configuration in \
+  .env docker-compose.yml README.md ROLLBACK.md; do
+  grep -Fx "$required_infra_configuration" \
+    <<<"$infra_configuration_members" >/dev/null || {
+      echo "missing protected infra recovery configuration: $required_infra_configuration" >&2
+      exit 65
+    }
+done
 configuration_members="$(tar -tzf "$payload_dir/configuration.tgz")"
 for required_configuration in \
-  .env docker-compose.yml Dockerfile.backend Dockerfile.mosquitto-reloader \
+  .env docker-compose.yml docker-compose.live.yml docker-compose.ci.yml \
+  docker-compose.infra-client.yml compose.phase4.yml \
+  Dockerfile.backend Dockerfile.mosquitto-reloader \
   anubis/botPolicy.yaml logging/prometheus.yml logging/alertmanager.yml \
-  backend/src/db/schema/base.sql scripts/backup.sh scripts/restore-drill.sh \
+  backend/src/db/schema/base.sql scripts/backup.sh scripts/restore-drill.sh scripts/sync-latest.sh \
   scripts/replace-container.sh; do
-  printf '%s\n' "$configuration_members" | grep -Fxq "$required_configuration" || {
+  grep -Fx "$required_configuration" <<<"$configuration_members" >/dev/null || {
     echo "missing protected recovery configuration: $required_configuration" >&2
     exit 65
   }
@@ -214,8 +226,8 @@ done
 mosquitto_configuration_members="$(tar -tzf "$payload_dir/mosquitto-config.tgz")"
 for required_mosquitto_configuration in \
   mosquitto/mosquitto.conf mosquitto/acl mosquitto/passwd; do
-  printf '%s\n' "$mosquitto_configuration_members" \
-    | grep -Fxq "$required_mosquitto_configuration" || {
+  grep -Fx "$required_mosquitto_configuration" \
+    <<<"$mosquitto_configuration_members" >/dev/null || {
       echo "missing Mosquitto recovery configuration: $required_mosquitto_configuration" >&2
       exit 65
     }
@@ -225,6 +237,7 @@ docker network create "$network_name" >/dev/null
 docker volume create "$postgres_volume" >/dev/null
 docker volume create "$redis_volume" >/dev/null
 docker volume create "$mosquitto_volume" >/dev/null
+docker volume create "$mosquitto_config_volume" >/dev/null
 
 docker run -d \
   --name "$postgres_name" \
@@ -237,13 +250,15 @@ docker run -d \
   "$timescale_image" \
   postgres -c autovacuum=off >/dev/null
 for _ in $(seq 1 240); do
-  if docker logs "$postgres_name" 2>&1 | grep -q 'PostgreSQL init process complete' &&
+  postgres_logs="$(docker logs "$postgres_name" 2>&1)"
+  if grep -q 'PostgreSQL init process complete' <<<"$postgres_logs" &&
     docker exec "$postgres_name" pg_isready -U meshcore -d meshcore >/dev/null 2>&1; then
     break
   fi
   sleep 0.5
 done
-docker logs "$postgres_name" 2>&1 | grep -q 'PostgreSQL init process complete'
+postgres_logs="$(docker logs "$postgres_name" 2>&1)"
+grep -q 'PostgreSQL init process complete' <<<"$postgres_logs"
 docker exec "$postgres_name" pg_isready -U meshcore -d meshcore >/dev/null
 docker exec "$postgres_name" createdb -U meshcore meshcore_owner_auth
 docker exec -i "$postgres_name" pg_restore \
@@ -255,11 +270,11 @@ docker exec -i "$postgres_name" pg_restore \
 echo "PostgreSQL datasets restored."
 
 echo "Restoring Redis durable state..."
-docker run --rm \
+docker run --rm -i \
   -v "$redis_volume:/restore" \
-  -v "$payload_dir:/payload:ro" \
   "$busybox_image" \
-  sh -c 'tar -C /restore -xzf /payload/redis-state.tgz' >/dev/null
+  sh -c 'tar -C /restore -xzf -' \
+  <"$payload_dir/redis-state.tgz" >/dev/null
 docker run -d \
   --name "$redis_name" \
   --network "$network_name" \
@@ -273,8 +288,9 @@ docker run -d \
   --maxmemory-policy noeviction >/dev/null
 redis_ready="false"
 for _ in $(seq 1 60); do
-  if docker exec "$redis_name" redis-cli -a "$restore_password" --no-auth-warning ping \
-    2>/dev/null | grep -q PONG; then
+  redis_ping="$(docker exec "$redis_name" redis-cli -a "$restore_password" --no-auth-warning ping \
+    2>/dev/null || true)"
+  if grep -Fxq PONG <<<"$redis_ping"; then
     redis_ready="true"
     break
   fi
@@ -302,17 +318,22 @@ done
 # uid. The parent remains 0700 on the host, while this signed, short-lived
 # child directory is traversable only through the container's read-only bind.
 chmod 0755 "$tmp_dir/mosquitto"
-docker run --rm \
+tar -C "$tmp_dir/mosquitto" -cf - mosquitto.conf acl passwd \
+  | docker run --rm -i \
+      -v "$mosquitto_config_volume:/restore" \
+      "$busybox_image" \
+      sh -c 'tar -C /restore -xf -' >/dev/null
+docker run --rm -i \
   -v "$mosquitto_volume:/restore" \
-  -v "$payload_dir:/payload:ro" \
   "$busybox_image" \
-  sh -c 'tar -C /restore -xzf /payload/mosquitto-state.tgz; chown -R 1883:1883 /restore' \
+  sh -c 'tar -C /restore -xzf -; chown -R 1883:1883 /restore' \
+  <"$payload_dir/mosquitto-state.tgz" \
   >/dev/null
 docker run -d \
   --name "$mosquitto_name" \
   --network "$network_name" \
   --network-alias mosquitto \
-  -v "$tmp_dir/mosquitto:/mosquitto/config:ro" \
+  -v "$mosquitto_config_volume:/mosquitto/config:ro" \
   -v "$mosquitto_volume:/mosquitto/data" \
   "$mosquitto_image" >/dev/null
 mosquitto_ready="false"
@@ -356,7 +377,7 @@ docker run -d \
   --cap-drop ALL \
   --security-opt no-new-privileges:true \
   --tmpfs /tmp:rw,noexec,nosuid,size=64m \
-  -v "$tmp_dir/mosquitto/acl:/mosquitto/config/acl:ro" \
+  -v "$mosquitto_config_volume:/mosquitto/config:ro" \
   -e "DATABASE_URL=$database_url" \
   -e "OWNER_DATABASE_URL=$owner_database_url" \
   -e "REDIS_URL=$redis_url" \
@@ -464,7 +485,7 @@ jq -n \
     archive_sha256: $archive_sha256,
     source_revision: $source_revision,
     schema_version: $schema_version,
-    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration"],
+    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"],
     evidence: {
       integrity: $integrity,
       owner_lookup_count: $owner_lookup_count

scripts/sync-latest.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+source_dir="${BACKUP_SOURCE_DIR:-/home/ben/ukmesh/backups}"
+target_dir="${BACKUP_SYNC_TARGET_DIR:?BACKUP_SYNC_TARGET_DIR must name the mounted off-host publication target}"
+verify_key="${BACKUP_RECEIPT_VERIFY_KEY:?BACKUP_RECEIPT_VERIFY_KEY is required}"
+allow_local="${BACKUP_SYNC_ALLOW_LOCAL_TEST:-false}"
+
+for command in openssl jq sha256sum findmnt find sort sed awk sync; do
+  command -v "$command" >/dev/null || {
+    echo "required command is unavailable: $command" >&2
+    exit 69
+  }
+done
+test -d "$source_dir" || { echo "backup source is missing: $source_dir" >&2; exit 66; }
+test -r "$verify_key" || { echo "receipt verify key is unreadable" >&2; exit 66; }
+mkdir -p "$target_dir"
+target_dir="$(cd "$target_dir" && pwd)"
+test -f "$target_dir/.meshcore-offsite-target" || {
+  echo "sync target is not attested (.meshcore-offsite-target missing)" >&2
+  exit 65
+}
+if [ "$allow_local" != "true" ] \
+  && [ "$(findmnt -T "$source_dir" -n -o SOURCE)" = "$(findmnt -T "$target_dir" -n -o SOURCE)" ]; then
+  echo "sync target shares the backup source filesystem" >&2
+  exit 65
+fi
+
+receipt_path="$(
+  find "$source_dir" -maxdepth 1 -type f -name 'backup-*.receipt.json' -printf '%T@ %p\n' \
+    | sort -nr \
+    | sed -n '1 { s/^[^ ]* //; p; }'
+)"
+test -n "$receipt_path" && test -f "$receipt_path" || {
+  echo "no completed backup receipt found" >&2
+  exit 66
+}
+signature_path="${receipt_path}.sig"
+test -f "$signature_path" || { echo "backup receipt signature is missing" >&2; exit 66; }
+openssl dgst -sha256 -verify "$verify_key" \
+  -signature "$signature_path" "$receipt_path" >/dev/null
+jq -e '.format == "meshcore-backup-receipt-v1" and .status == "complete"' \
+  "$receipt_path" >/dev/null
+archive_name="$(jq -er '.archive' "$receipt_path")"
+case "$archive_name" in
+  */*|'') echo "backup receipt archive name is unsafe" >&2; exit 65 ;;
+esac
+archive_path="$source_dir/$archive_name"
+test -f "$archive_path" || { echo "backup archive is missing" >&2; exit 66; }
+expected_archive_sha="$(jq -er '.archive_sha256' "$receipt_path")"
+actual_archive_sha="$(sha256sum "$archive_path" | awk '{print $1}')"
+test "$actual_archive_sha" = "$expected_archive_sha" || {
+  echo "backup archive checksum does not match receipt" >&2
+  exit 65
+}
+
+publication_id="$(jq -er '.backup_id' "$receipt_path")"
+case "$publication_id" in
+  backup-[0-9A-Za-z._-]*) ;;
+  *) echo "backup id is unsafe" >&2; exit 65 ;;
+esac
+partial_dir="$target_dir/.${publication_id}.partial"
+test ! -e "$partial_dir" || { echo "partial publication already exists" >&2; exit 73; }
+mkdir "$partial_dir"
+cleanup() {
+  if [ -d "$partial_dir" ]; then
+    find "$partial_dir" -maxdepth 1 -type f -delete
+    rmdir "$partial_dir"
+  fi
+}
+trap cleanup EXIT
+
+cp -- "$archive_path" "$partial_dir/$archive_name"
+cp -- "$receipt_path" "$partial_dir/latest.receipt.json"
+cp -- "$signature_path" "$partial_dir/latest.receipt.json.sig"
+test "$(sha256sum "$partial_dir/$archive_name" | awk '{print $1}')" = "$expected_archive_sha"
+openssl dgst -sha256 -verify "$verify_key" \
+  -signature "$partial_dir/latest.receipt.json.sig" \
+  "$partial_dir/latest.receipt.json" >/dev/null
+(
+  cd "$partial_dir"
+  sha256sum "$archive_name" latest.receipt.json latest.receipt.json.sig \
+    >latest.sha256
+  sha256sum --check latest.sha256 >/dev/null
+)
+sync "$partial_dir/$archive_name" "$partial_dir/latest.receipt.json" \
+  "$partial_dir/latest.receipt.json.sig" "$partial_dir/latest.sha256"
+
+mv "$partial_dir/$archive_name" "$target_dir/$archive_name"
+mv "$partial_dir/latest.receipt.json" "$target_dir/latest.receipt.json"
+mv "$partial_dir/latest.receipt.json.sig" "$target_dir/latest.receipt.json.sig"
+mv "$partial_dir/latest.sha256" "$target_dir/latest.sha256"
+rmdir "$partial_dir"
+sync "$target_dir/$archive_name" "$target_dir/latest.receipt.json" \
+  "$target_dir/latest.receipt.json.sig" "$target_dir/latest.sha256"
+trap - EXIT
+
+(
+  cd "$target_dir"
+  sha256sum --check latest.sha256 >/dev/null
+)
+openssl dgst -sha256 -verify "$verify_key" \
+  -signature "$target_dir/latest.receipt.json.sig" \
+  "$target_dir/latest.receipt.json" >/dev/null
+jq -n --arg backup_id "$publication_id" --arg target "$target_dir" \
+  --arg archive "$archive_name" \
+  '{backup_id: $backup_id, target: $target, archive: $archive, status: "verified"}'