ukmesh/logging/rules/meshcore.yml

groups:
  - name: meshcore-availability
    interval: 15s
    rules:
      - alert: BackendDown
        expr: up{job="meshcore-backend"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: MeshCore backend metrics endpoint is unavailable
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#backenddown

      - alert: BackendNotReady
        expr: probe_success{job="meshcore-readiness"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: MeshCore dependency readiness probe is failing
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#backendnotready

      - alert: SyntheticProbeFailed
        expr: probe_success{job="meshcore-synthetic"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: A public application synthetic probe is failing
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#syntheticprobefailed

      - alert: DependencyExporterDown
        expr: up{job=~"postgres|redis"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: Dependency exporter {{ $labels.job }} is unavailable
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#dependencyexporterdown

      - alert: MosquittoUnavailable
        expr: probe_success{job="mosquitto-broker"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: Mosquitto WebSocket listener is unavailable
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#mosquittounavailable

      - alert: MeshIngestSilent
        expr: (sum(rate(meshcore_mqtt_messages_total[15m])) or vector(0)) == 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: No accepted MeshCore MQTT packets for 30 minutes
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#meshingestsilent

  - name: meshcore-queues-and-workers
    interval: 15s
    rules:
      - alert: QueueAgeHigh
        expr: meshcore_link_queue_oldest_age_seconds > 600
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: A durable worker queue has jobs older than ten minutes
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#queueagehigh

      - alert: QueueCapacityHigh
        expr: meshcore_link_queue_active_jobs > 4000 or meshcore_link_queue_active_bytes > 53687091
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: A durable queue is above 80 percent of its configured bound
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#queuecapacityhigh

      - alert: DeadJobsPresent
        expr: meshcore_link_queue_dead_jobs > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: A durable worker queue contains dead-letter jobs
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#deadjobspresent

      - alert: AnalysisLeaseLost
        expr: increase(meshcore_analysis_lease_events_total{outcome=~"heartbeat_lost|finish_lease_lost"}[10m]) > 0
        labels:
          severity: critical
        annotations:
          summary: Analysis workload {{ $labels.workload }} lost its publication lease
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#analysisleaselost

      - alert: HealthWorkerHeartbeatStale
        expr: meshcore_worker_heartbeat_age_seconds{worker="health"} < 0 or meshcore_worker_heartbeat_age_seconds{worker="health"} > 180
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: Health worker heartbeat is stale or missing
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#healthworkerheartbeatstale

      - alert: WorkerMetricsDown
        expr: up{job="meshcore-workers"} == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: Required worker {{ $labels.role }} is unavailable
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#workermetricsdown

      - alert: ActiveQueueWorkerHeartbeatStale
        # Redis worker heartbeat: ten-second cadence, 45-second TTL. Missing
        # heartbeat is unhealthy too; ITM writes may legitimately pause for 15m.
        # Compare each reporter's own queue and heartbeat. A failed Redis read
        # on a different exporter must not page for this reporter's healthy queue.
        expr: meshcore_link_queue_active_jobs > 0 and on(job, instance) (meshcore_worker_heartbeat_age_seconds{worker="link"} < 0 or meshcore_worker_heartbeat_age_seconds{worker="link"} > 180)
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: A worker with queued work has a stale heartbeat
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#activequeueworkerheartbeatstale

  - name: meshcore-database-and-host
    interval: 30s
    rules:
      - alert: DatabaseUnavailable
        expr: pg_up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: PostgreSQL exporter cannot query the database
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databaseunavailable

      - alert: DatabasePoolSaturated
        expr: meshcore_db_pool_connections{state=~".*_waiting"} > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Application database pool has waiting callers
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databasepoolsaturated

      - alert: DatabaseVacuumBacklog
        # Keep the SQL health gate and the exporter alert aligned. The
        # absolute branch catches dead-only Timescale chunks whose live-row
        # estimate has already fallen below the relative threshold.
        expr: |
          pg_stat_user_tables_n_dead_tup > 50000
          or (
            pg_stat_user_tables_n_live_tup > 10000
            and pg_stat_user_tables_n_dead_tup > 10000
            and pg_stat_user_tables_n_dead_tup > pg_stat_user_tables_n_live_tup * 0.1
          )
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: PostgreSQL table {{ $labels.relname }} has a vacuum backlog
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databasevacuumbacklog

      - alert: DatabaseWalRateHigh
        expr: rate(pg_wal_bytes[15m]) > 52428800
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: PostgreSQL WAL generation exceeds 50 MiB per second
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databasewalratehigh

      - alert: HostDiskSpaceLow
        expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} < 0.10
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: Host filesystem {{ $labels.mountpoint }} has less than 10 percent free
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#hostdiskspacelow

      - alert: HostDiskExhaustionForecast
        expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}[6h], 86400) < 0
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: Host filesystem {{ $labels.mountpoint }} is forecast to fill within 24 hours
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#hostdiskexhaustionforecast

      - alert: CoreTelemetryGrowthHigh
        expr: sum by (table) (deriv(meshcore_core_telemetry_hypertable_bytes{role="health"}[7d]) * 604800) > 1073741824
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: Core telemetry hypertable {{ $labels.table }} is growing by over 1 GiB per week
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#coretelemetrygrowthhigh

      - alert: CoreTelemetryRetentionOverdue
        expr: meshcore_core_telemetry_expired_chunks{role="health"} > 0
        for: 27h
        labels:
          severity: critical
        annotations:
          summary: Core telemetry hypertable {{ $labels.table }} has expired chunks still present
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#coretelemetryretentionoverdue

      - alert: CoreTelemetryCompressionOverdue
        expr: meshcore_core_telemetry_overdue_uncompressed_chunks{role="health"} > 0
        for: 3h
        labels:
          severity: warning
        annotations:
          summary: Core telemetry hypertable {{ $labels.table }} has chunks overdue for compression
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#coretelemetrycompressionoverdue

      - alert: PacketPathsGrowthHigh
        expr: meshcore_packet_paths_rows_30d{role="health"} / 30 > 30000
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: Durable packet path growth exceeds 30,000 rows per day
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#packetpathscapacity

      - alert: PacketPathsCompressionOverdue
        expr: meshcore_packet_paths_overdue_uncompressed_chunks{role="health"} > 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: One or more durable packet path chunks missed compression
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#packetpathscapacity

  - name: meshcore-data-integrity
    interval: 30s
    rules:
      - alert: BackupReceiptMissing
        expr: count(meshcore_backup_age_seconds{dataset=~"analytics|owner_auth|mosquitto"}) < 3
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: One or more required verified backup receipts are missing
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#backupreceiptmissing

      - alert: BackupStale
        expr: meshcore_backup_age_seconds{dataset=~"analytics|owner_auth|mosquitto"} > 86400
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: Verified {{ $labels.dataset }} backup receipt is older than 24 hours
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#backupstale

      - alert: TelemetryIntegrityFailures
        expr: increase(meshcore_privacy_filter_total{outcome="invalid"}[10m]) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Packet visibility validation is rejecting elevated telemetry
          owner: meshcore-operations
          runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#telemetryintegrityfailures