mirror of
https://github.com/gadgethd/ukmesh.git
synced 2026-10-22 05:52:29 +00:00
hermes-gadgetandgadgethd
fa801dc004
* fix(backlog): close ukmesh health and advert identity gaps * docs(review): record backlog round 2 pull request --------- Co-authored-by: gadgethd <111318106+gadgethd@users.noreply.github.com>
373 lines
15 KiB
YAML
373 lines
15 KiB
YAML
rule_files:
|
|
- meshcore.yml
|
|
|
|
evaluation_interval: 1m
|
|
|
|
tests:
|
|
- name: a failed backend heartbeat read cannot page for a healthy health-worker queue
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'meshcore_link_queue_active_jobs{job="meshcore-workers",instance="health-worker:9091",role="health"}'
|
|
values: 12x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{job="meshcore-workers",instance="health-worker:9091",role="health",worker="link"}'
|
|
values: 5x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{job="meshcore-backend",instance="backend:9091",worker="link"}'
|
|
values: -1x8
|
|
alert_rule_test:
|
|
- eval_time: 8m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: queued work and stale heartbeat must come from the same reporter instance
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'meshcore_link_queue_active_jobs{job="meshcore-workers",instance="healthy:9091",role="health"}'
|
|
values: 12x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{job="meshcore-workers",instance="healthy:9091",role="health",worker="link"}'
|
|
values: 5x8
|
|
- series: 'meshcore_link_queue_active_jobs{job="meshcore-workers",instance="idle:9091",role="health"}'
|
|
values: 0x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{job="meshcore-workers",instance="idle:9091",role="health",worker="link"}'
|
|
values: -1x8
|
|
alert_rule_test:
|
|
- eval_time: 8m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: a stale queued reporter pages independently and clears on its own heartbeat recovery
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'meshcore_link_queue_active_jobs{job="meshcore-workers",instance="health-worker:9091",role="health"}'
|
|
values: 12x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{job="meshcore-workers",instance="health-worker:9091",role="health",worker="link"}'
|
|
values: -1x5 5x3
|
|
- series: 'meshcore_link_queue_active_jobs{job="meshcore-backend",instance="backend:9091"}'
|
|
values: 12x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{job="meshcore-backend",instance="backend:9091",worker="link"}'
|
|
values: 5x8
|
|
alert_rule_test:
|
|
- eval_time: 2m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
- eval_time: 4m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts:
|
|
- exp_labels:
|
|
job: meshcore-workers
|
|
instance: health-worker:9091
|
|
role: health
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: A worker with queued work has a stale heartbeat
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#activequeueworkerheartbeatstale
|
|
- eval_time: 6m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: fresh real link heartbeats stay healthy throughout a fifteen-minute ITM batch gap
|
|
interval: 1m
|
|
input_series:
|
|
- series: meshcore_link_queue_active_jobs
|
|
values: 12x20
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{worker="link"}'
|
|
values: 5x20
|
|
alert_rule_test:
|
|
- eval_time: 20m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: expired link heartbeat with queued work alerts after holdoff and recovers
|
|
interval: 1m
|
|
input_series:
|
|
- series: meshcore_link_queue_active_jobs
|
|
values: 12x9
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{worker="link"}'
|
|
values: 5 -1 -1 -1 -1 -1 5 5 5 5
|
|
alert_rule_test:
|
|
- eval_time: 3m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
- eval_time: 4m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: A worker with queued work has a stale heartbeat
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#activequeueworkerheartbeatstale
|
|
- eval_time: 6m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: a missing link heartbeat without queued work does not page
|
|
interval: 1m
|
|
input_series:
|
|
- series: meshcore_link_queue_active_jobs
|
|
values: 0x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{worker="link"}'
|
|
values: -1x8
|
|
alert_rule_test:
|
|
- eval_time: 8m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: a brief missing heartbeat does not outlast the three-minute holdoff
|
|
interval: 1m
|
|
input_series:
|
|
- series: meshcore_link_queue_active_jobs
|
|
values: 12x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{worker="link"}'
|
|
values: 5 -1 -1 5 5 5 5 5 5
|
|
alert_rule_test:
|
|
- eval_time: 8m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts: []
|
|
|
|
- name: bounded queues and stopped workers produce their documented alerts
|
|
interval: 1m
|
|
input_series:
|
|
- series: meshcore_link_queue_oldest_age_seconds
|
|
values: 700x8
|
|
- series: meshcore_link_queue_active_jobs
|
|
values: 4500x8
|
|
- series: meshcore_link_queue_dead_jobs
|
|
values: 1x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{worker="health"}'
|
|
values: 200x8
|
|
- series: 'up{job="meshcore-workers",role="link"}'
|
|
values: 0x8
|
|
- series: 'meshcore_worker_heartbeat_age_seconds{worker="link"}'
|
|
values: 200x8
|
|
alert_rule_test:
|
|
- eval_time: 6m
|
|
alertname: QueueAgeHigh
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: A durable worker queue has jobs older than ten minutes
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#queueagehigh
|
|
- eval_time: 6m
|
|
alertname: QueueCapacityHigh
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: A durable queue is above 80 percent of its configured bound
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#queuecapacityhigh
|
|
- eval_time: 6m
|
|
alertname: DeadJobsPresent
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: A durable worker queue contains dead-letter jobs
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#deadjobspresent
|
|
- eval_time: 4m
|
|
alertname: HealthWorkerHeartbeatStale
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: critical
|
|
worker: health
|
|
exp_annotations:
|
|
summary: Health worker heartbeat is stale or missing
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#healthworkerheartbeatstale
|
|
- eval_time: 4m
|
|
alertname: WorkerMetricsDown
|
|
exp_alerts:
|
|
- exp_labels:
|
|
job: meshcore-workers
|
|
role: link
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Required worker link is unavailable
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#workermetricsdown
|
|
- eval_time: 4m
|
|
alertname: ActiveQueueWorkerHeartbeatStale
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: A worker with queued work has a stale heartbeat
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#activequeueworkerheartbeatstale
|
|
|
|
- name: database host and backup failures produce their documented alerts
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'pg_up{instance="database"}'
|
|
values: 0x14
|
|
- series: 'meshcore_db_pool_connections{state="primary_waiting"}'
|
|
values: 1x14
|
|
- series: 'pg_stat_user_tables_n_live_tup{relname="dead_only"}'
|
|
values: 1x14
|
|
- series: 'pg_stat_user_tables_n_dead_tup{relname="dead_only"}'
|
|
values: 60000x14
|
|
- series: 'node_filesystem_avail_bytes{fstype="ext4",mountpoint="/data"}'
|
|
values: 5x14
|
|
- series: 'node_filesystem_size_bytes{fstype="ext4",mountpoint="/data"}'
|
|
values: 100x14
|
|
- series: 'node_filesystem_avail_bytes{fstype="ext4",mountpoint="/forecast"}'
|
|
values: 10000 9900 9800 9700 9600 9500 9400 9300 9200 9100 9000 8900 8800 8700 8600 8500 8400 8300 8200 8100 8000
|
|
- series: 'meshcore_backup_age_seconds{dataset="analytics"}'
|
|
values: 90000x14
|
|
- series: 'meshcore_backup_age_seconds{dataset="owner_auth"}'
|
|
values: 10x14
|
|
alert_rule_test:
|
|
- eval_time: 3m
|
|
alertname: DatabaseUnavailable
|
|
exp_alerts:
|
|
- exp_labels:
|
|
instance: database
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: PostgreSQL exporter cannot query the database
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databaseunavailable
|
|
- eval_time: 6m
|
|
alertname: DatabasePoolSaturated
|
|
exp_alerts:
|
|
- exp_labels:
|
|
state: primary_waiting
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: Application database pool has waiting callers
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databasepoolsaturated
|
|
- eval_time: 11m
|
|
alertname: DatabaseVacuumBacklog
|
|
exp_alerts:
|
|
- exp_labels:
|
|
relname: dead_only
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: PostgreSQL table dead_only has a vacuum backlog
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#databasevacuumbacklog
|
|
- eval_time: 11m
|
|
alertname: HostDiskSpaceLow
|
|
exp_alerts:
|
|
- exp_labels:
|
|
fstype: ext4
|
|
mountpoint: /data
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Host filesystem /data has less than 10 percent free
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#hostdiskspacelow
|
|
- eval_time: 20m
|
|
alertname: HostDiskExhaustionForecast
|
|
exp_alerts:
|
|
- exp_labels:
|
|
fstype: ext4
|
|
mountpoint: /forecast
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Host filesystem /forecast is forecast to fill within 24 hours
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#hostdiskexhaustionforecast
|
|
- eval_time: 11m
|
|
alertname: BackupReceiptMissing
|
|
exp_alerts:
|
|
- exp_labels:
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: One or more required verified backup receipts are missing
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#backupreceiptmissing
|
|
- eval_time: 11m
|
|
alertname: BackupStale
|
|
exp_alerts:
|
|
- exp_labels:
|
|
dataset: analytics
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Verified analytics backup receipt is older than 24 hours
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#backupstale
|
|
|
|
- name: lease and privacy integrity counters alert on observed failures
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'meshcore_analysis_lease_events_total{outcome="heartbeat_lost",workload="stats"}'
|
|
values: 0 0 1 1 1 1 1 1
|
|
- series: 'meshcore_privacy_filter_total{outcome="invalid"}'
|
|
values: 0 0 0 20 20 20 20 20
|
|
alert_rule_test:
|
|
- eval_time: 3m
|
|
alertname: AnalysisLeaseLost
|
|
exp_alerts:
|
|
- exp_labels:
|
|
outcome: heartbeat_lost
|
|
workload: stats
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Analysis workload stats lost its publication lease
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#analysisleaselost
|
|
- eval_time: 8m
|
|
alertname: TelemetryIntegrityFailures
|
|
exp_alerts:
|
|
- exp_labels:
|
|
outcome: invalid
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: Packet visibility validation is rejecting elevated telemetry
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#telemetryintegrityfailures
|
|
|
|
- name: telemetry growth forecast
|
|
interval: 1d
|
|
input_series:
|
|
- series: 'meshcore_core_telemetry_hypertable_bytes{role="health",table="packets"}'
|
|
values: 1000000000 3000000000 5000000000 7000000000 9000000000 11000000000 13000000000 15000000000 17000000000
|
|
alert_rule_test:
|
|
- eval_time: 8d1h
|
|
alertname: CoreTelemetryGrowthHigh
|
|
exp_alerts:
|
|
- exp_labels:
|
|
table: packets
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: Core telemetry hypertable packets is growing by over 1 GiB per week
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#coretelemetrygrowthhigh
|
|
|
|
- name: telemetry lifecycle coverage
|
|
interval: 1m
|
|
input_series:
|
|
- series: 'meshcore_core_telemetry_expired_chunks{role="health",table="node_status_samples"}'
|
|
values: 0 1x1800
|
|
- series: 'meshcore_core_telemetry_overdue_uncompressed_chunks{role="health",table="node_neighbor_samples"}'
|
|
values: 1x360
|
|
alert_rule_test:
|
|
- eval_time: 29h
|
|
alertname: CoreTelemetryRetentionOverdue
|
|
exp_alerts:
|
|
- exp_labels:
|
|
role: health
|
|
table: node_status_samples
|
|
severity: critical
|
|
exp_annotations:
|
|
summary: Core telemetry hypertable node_status_samples has expired chunks still present
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#coretelemetryretentionoverdue
|
|
- eval_time: 4h
|
|
alertname: CoreTelemetryCompressionOverdue
|
|
exp_alerts:
|
|
- exp_labels:
|
|
role: health
|
|
table: node_neighbor_samples
|
|
severity: warning
|
|
exp_annotations:
|
|
summary: Core telemetry hypertable node_neighbor_samples has chunks overdue for compression
|
|
owner: meshcore-operations
|
|
runbook_url: https://github.com/gadgethd/ukmesh/blob/main/docs/operations.md#coretelemetrycompressionoverdue
|