ukmesh/scripts/backup.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
project_dir="$(cd -- "${script_dir}/.." && pwd)"
app_project_name="${COMPOSE_PROJECT_NAME:-meshcore-analytics}"
infra_project_dir="${MESHCORE_INFRA_DIR:-${project_dir}/../meshcore-infra}"
infra_project_name="${MESHCORE_INFRA_PROJECT_NAME:-meshcore-infra}"
output_dir="${BACKUP_OUTPUT_DIR:?BACKUP_OUTPUT_DIR must name an encrypted backup target}"
encryption_cert="${BACKUP_ENCRYPTION_CERT:?BACKUP_ENCRYPTION_CERT must name a public X.509 certificate}"
signing_key="${BACKUP_RECEIPT_SIGNING_KEY:?BACKUP_RECEIPT_SIGNING_KEY must name the offline receipt signing key}"
allow_local="${BACKUP_ALLOW_LOCAL_STAGING:-false}"
minimum_free_bytes="${BACKUP_MIN_FREE_BYTES:-10737418240}"
db_user="${POSTGRES_USER:-meshcore}"
analytics_db="${POSTGRES_DB:-meshcore}"
owner_db="${OWNER_POSTGRES_DB:-meshcore_owner_auth}"
busybox_image="busybox@sha256:9532d8c39891ca2ecde4d30d7710e01fb739c87a8b9299685c63704296b16028"

for command in docker openssl jq tar sha256sum stat findmnt df find sort split; do
  command -v "$command" >/dev/null || {
    echo "required command is unavailable: $command" >&2
    exit 69
  }
done
for path in "$encryption_cert" "$signing_key"; do
  test -r "$path" || {
    echo "required key material is not readable: $path" >&2
    exit 66
  }
done
test -f "$infra_project_dir/docker-compose.yml" || {
  echo "meshcore-infra compose file is missing: $infra_project_dir/docker-compose.yml" >&2
  exit 66
}
openssl x509 -in "$encryption_cert" -noout >/dev/null
openssl pkey -in "$signing_key" -noout >/dev/null

mkdir -p "$output_dir"
output_dir="$(cd "$output_dir" && pwd)"
project_device="$(findmnt -T "$project_dir" -n -o SOURCE)"
output_device="$(findmnt -T "$output_dir" -n -o SOURCE)"
if [ "$allow_local" != "true" ]; then
  test -f "$output_dir/.meshcore-offsite-target" || {
    echo "backup target is not attested as off-host (.meshcore-offsite-target missing)" >&2
    exit 65
  }
  if [ "$project_device" = "$output_device" ]; then
    echo "backup target shares the application filesystem; use a mounted off-host target" >&2
    exit 65
  fi
fi
free_bytes="$(df -PB1 "$output_dir" | awk 'NR==2 {print $4}')"
if [ "$free_bytes" -lt "$minimum_free_bytes" ]; then
  echo "backup target has insufficient free space" >&2
  exit 75
fi

cd "$project_dir"
if [ ! -f "$project_dir/.env" ] || [ -L "$project_dir/.env" ]; then
  echo "protected runtime configuration .env is missing or unsafe" >&2
  exit 66
fi
env_mode="$(stat -c '%a' "$project_dir/.env")"
if (( (8#$env_mode & 8#077) != 0 )); then
  echo "protected runtime configuration .env must not be group/world accessible" >&2
  exit 66
fi
app_compose=(
  docker compose --project-directory "$project_dir"
  -f "$project_dir/docker-compose.yml"
  -f "$project_dir/docker-compose.live.yml"
  --project-name "$app_project_name"
)
infra_compose=(
  docker compose --project-directory "$infra_project_dir"
  -f "$infra_project_dir/docker-compose.yml"
  --project-name "$infra_project_name"
)
"${app_compose[@]}" config -q
"${infra_compose[@]}" config -q
declare -A service_container_ids=()
for service in timescaledb redis mosquitto; do
  container_id="$("${infra_compose[@]}" ps -q "$service")"
  test -n "$container_id" || {
    echo "required service is not running: $service" >&2
    exit 69
  }
  if [ "$(docker inspect "$container_id" --format '{{.State.Status}}')" != "running" ]; then
    echo "required service is not running: $service" >&2
    exit 69
  fi
  service_container_ids["$service"]="$container_id"
done
timescaledb_container="${service_container_ids[timescaledb]}"
redis_container="${service_container_ids[redis]}"
mosquitto_container="${service_container_ids[mosquitto]}"

started_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)"
backup_id="backup-$(date -u +%Y%m%dT%H%M%SZ)-$(git rev-parse --short=12 HEAD)"
source_revision="$(git rev-parse HEAD)"
tmp_dir="$(mktemp -d)"
payload_dir="$tmp_dir/payload"
mkdir -p "$payload_dir"
archive_partial=""
archive_path=""
receipt_partial=""
receipt_path=""
signature_partial=""
backup_complete="false"
cleanup() {
  rm -rf -- "$tmp_dir"
  if [ "$backup_complete" != "true" ]; then
    for incomplete_path in \
      "$archive_partial" \
      "$archive_path" \
      "$receipt_partial" \
      "$receipt_path" \
      "$signature_partial" \
      "${receipt_path:+${receipt_path}.sig}"; do
      if [ -n "$incomplete_path" ]; then
        rm -f -- "$incomplete_path"
      fi
    done
  fi
}
trap cleanup EXIT

echo "Creating consistent PostgreSQL custom-format dumps..."
docker exec -i "$timescaledb_container" \
  pg_dump -U "$db_user" -d "$analytics_db" \
  --format=custom --compress=6 --no-owner --no-acl \
  >"$payload_dir/analytics.dump"
docker exec -i "$timescaledb_container" \
  pg_dump -U "$db_user" -d "$owner_db" \
  --format=custom --compress=6 --no-owner --no-acl \
  >"$payload_dir/owner-auth.dump"

schema_version="$(
  docker exec -i "$timescaledb_container" \
    psql -U "$db_user" -d "$analytics_db" -Atc \
    "SELECT COALESCE(MAX(((regexp_match(name, '^([0-9]+)_'))[1])::int), 0) FROM schema_migrations"
)"
schema_version="${schema_version//$'\r'/}"

echo "Checkpointing Redis durable queues and state..."
docker exec -i "$redis_container" sh -c \
  'redis-cli -a "$REDIS_PASSWORD" --no-auth-warning BGSAVE >/dev/null || true'
redis_ready="false"
for _ in $(seq 1 120); do
  persistence="$(
    docker exec -i "$redis_container" sh -c \
      'redis-cli -a "$REDIS_PASSWORD" --no-auth-warning INFO persistence'
  )"
  persistence="${persistence//$'\r'/}"
  if grep -q '^rdb_bgsave_in_progress:0$' <<<"$persistence" \
    && grep -q '^rdb_last_bgsave_status:ok$' <<<"$persistence"; then
    redis_ready="true"
    break
  fi
  sleep 0.5
done
test "$redis_ready" = "true" || {
  echo "Redis persistence checkpoint did not complete successfully" >&2
  exit 75
}
docker exec -i "$redis_container" \
  tar -C /data -czf - . >"$payload_dir/redis-state.tgz"

echo "Flushing and capturing Mosquitto credentials, ACL, and persistence..."
docker exec -i "$mosquitto_container" sh -c \
  'kill -USR1 1; sleep 1; tar -C /mosquitto/data -czf - .' \
  >"$payload_dir/mosquitto-state.tgz"
mosquitto_config_stage="$tmp_dir/mosquitto-configuration"
mkdir -p "$mosquitto_config_stage/mosquitto"
docker exec -i "$mosquitto_container" \
  tar -C /mosquitto/config -cf - mosquitto.conf acl passwd \
  | tar -C "$mosquitto_config_stage/mosquitto" -xf -
tar -C "$mosquitto_config_stage" -czf "$payload_dir/mosquitto-config.tgz" \
  mosquitto/mosquitto.conf mosquitto/acl mosquitto/passwd
tar -C "$project_dir" -czf "$payload_dir/configuration.tgz" \
  .env .env.example .trivyignore.yaml \
  docker-compose.yml docker-compose.live.yml docker-compose.ci.yml \
  docker-compose.infra-client.yml compose.phase4.yml \
  Dockerfile Dockerfile.app Dockerfile.backend Dockerfile.website \
  Dockerfile.mesh-health-check Dockerfile.mosquitto-reloader \
  viewshed-worker/Dockerfile \
  nginx.app.conf nginx.website.conf nginx.security-headers.conf \
  anubis/botPolicy.yaml docker/mosquitto-reloader.py \
  logging \
  backend/src/db/migrations backend/src/db/schema backend/src/db/owner-auth.sql \
  scripts/backup.sh scripts/restore-drill.sh scripts/sync-latest.sh scripts/replace-container.sh \
  scripts/bootstrap-mosquitto.sh vacuum-compressed-chunks.sh
tar -C "$infra_project_dir" -czf "$payload_dir/infra-configuration.tgz" \
  .env docker-compose.yml README.md ROLLBACK.md

completed_at="$(date -u +%Y-%m-%dT%H:%M:%S.000Z)"
manifest_path="$payload_dir/manifest.json"
manifest_files='{}'
while IFS= read -r -d '' payload_file; do
  payload_name="${payload_file##*/}"
  payload_sha256="$(sha256sum "$payload_file" | awk '{print $1}')"
  payload_bytes="$(stat -c '%s' "$payload_file")"
  manifest_files="$(
    jq \
      --arg name "$payload_name" \
      --arg sha256 "$payload_sha256" \
      --argjson bytes "$payload_bytes" \
      '. + {($name): {sha256: $sha256, bytes: $bytes}}' \
      <<<"$manifest_files"
  )"
done < <(
  find "$payload_dir" -maxdepth 1 -type f ! -name manifest.json -print0 \
    | sort -z
)
jq -n \
  --arg backup_id "$backup_id" \
  --arg source_revision "$source_revision" \
  --arg started_at "$started_at" \
  --arg completed_at "$completed_at" \
  --argjson schema_version "$schema_version" \
  --argjson files "$manifest_files" \
  '{
    format: "meshcore-backup-manifest-v1",
    backup_id: $backup_id,
    source_revision: $source_revision,
    started_at: $started_at,
    completed_at: $completed_at,
    schema_version: $schema_version,
    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"],
    files: $files
  }' >"$manifest_path"

archive_partial="$output_dir/${backup_id}.tar.gz.cms.partial"
archive_path="$output_dir/${backup_id}.tar.gz.cms"
encrypted_parts_dir="$tmp_dir/encrypted-parts"
mkdir -p "$encrypted_parts_dir"
export MESHCORE_BACKUP_ENCRYPTION_CERT="$encryption_cert"
tar -C "$payload_dir" -czf - . \
  | split \
      --bytes=134217728 \
      --numeric-suffixes=0 \
      --suffix-length=4 \
      --filter='openssl cms -encrypt -binary -aes-256-cbc -in - -outform DER -out "${FILE}.cms" "$MESHCORE_BACKUP_ENCRYPTION_CERT"' \
      - "$encrypted_parts_dir/part-"
unset MESHCORE_BACKUP_ENCRYPTION_CERT

printf 'MESHCORE-CMS-CHUNKS-v1\n' >"$archive_partial"
encrypted_part_count=0
for encrypted_part in "$encrypted_parts_dir"/part-*.cms; do
  test -f "$encrypted_part" || {
    echo "backup encryption did not produce any chunks" >&2
    exit 75
  }
  encrypted_part_bytes="$(stat -c '%s' "$encrypted_part")"
  printf '%020d\n' "$encrypted_part_bytes" >>"$archive_partial"
  cat "$encrypted_part" >>"$archive_partial"
  rm -f -- "$encrypted_part"
  encrypted_part_count="$((encrypted_part_count + 1))"
done
test "$encrypted_part_count" -gt 0
printf '%020d\n' 0 >>"$archive_partial"
archive_sha256="$(sha256sum "$archive_partial" | awk '{print $1}')"
archive_bytes="$(stat -c '%s' "$archive_partial")"

receipt_path="$output_dir/${backup_id}.receipt.json"
receipt_partial="${receipt_path}.partial"
jq -n \
  --arg backup_id "$backup_id" \
  --arg source_revision "$source_revision" \
  --arg started_at "$started_at" \
  --arg completed_at "$completed_at" \
  --arg archive "$(basename "$archive_path")" \
  --arg archive_sha256 "$archive_sha256" \
  --argjson archive_bytes "$archive_bytes" \
  --argjson schema_version "$schema_version" \
  '{
    format: "meshcore-backup-receipt-v1",
    backup_id: $backup_id,
    source_revision: $source_revision,
    started_at: $started_at,
    completed_at: $completed_at,
    schema_version: $schema_version,
    archive: $archive,
    archive_sha256: $archive_sha256,
    archive_bytes: $archive_bytes,
    datasets: ["analytics", "owner_auth", "mosquitto", "redis", "configuration", "infra_configuration"],
    encryption: "CMS-AES-256-CBC-CHUNKED-v1",
    status: "complete"
  }' >"$receipt_partial"
signature_partial="${receipt_path}.sig.partial"
openssl dgst -sha256 -sign "$signing_key" \
  -out "$signature_partial" "$receipt_partial"

mv "$archive_partial" "$archive_path"
mv "$receipt_partial" "$receipt_path"
mv "$signature_partial" "${receipt_path}.sig"
sync "$archive_path" "$receipt_path" "${receipt_path}.sig"
backup_complete="true"

echo "Encrypted backup complete:"
jq -n \
  --arg backup_id "$backup_id" \
  --arg archive "$archive_path" \
  --arg receipt "$receipt_path" \
  --argjson bytes "$archive_bytes" \
  '{backup_id: $backup_id, archive: $archive, receipt: $receipt, encrypted_bytes: $bytes}'