mirror of
https://github.com/gadgethd/ukmesh.git
synced 2026-10-23 06:12:27 +00:00
fix(health): align vacuum backlog with PostgreSQL triggers
This commit is contained in:
hermes-gadget committed
1 parent
bec753b5e9
commit
767f37e042
14 files changed
+1668
-861
No files matched your search
@@ -43,6 +43,7 @@ jobs:
|
||||
env:
|
||||
TEST_REDIS_URL: redis://127.0.0.1:6379
|
||||
TEST_ANALYSIS_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres
|
||||
TEST_VACUUM_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres
|
||||
TEST_INGEST_DATABASE_URL: postgresql://postgres:analysis-test-password@127.0.0.1:55433/postgres
|
||||
defaults:
|
||||
run:
|
||||
@@ -61,6 +62,7 @@ jobs:
|
||||
- run: npm run contract:check
|
||||
- run: npm test
|
||||
- run: npm run test:analysis-integration
|
||||
- run: npm run test:vacuum-integration
|
||||
- name: Configure bounded RAM Redis fixture
|
||||
run: |
|
||||
docker exec '${{ job.services.redis.id }}' redis-cli CONFIG SET maxmemory 96mb
|
||||
|
||||
@@ -12,6 +12,9 @@ test-results/
|
||||
# Local semantic code index
|
||||
.tokensave/
|
||||
|
||||
# Worktree-local verification tools and package cache
|
||||
.ukmesh-tools/
|
||||
|
||||
# Mosquitto credentials (generated file, not for source control)
|
||||
mosquitto/passwd
|
||||
mosquitto/acl
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"load:realtime": "tsx src/tools/loadRealtime.ts",
|
||||
"test": "node --import tsx --test $(find src -name '*.test.ts' ! -name '*.integration.test.ts' -print) && bash ../scripts/test-alert-receiver.sh",
|
||||
"test:analysis-integration": "node --import tsx --test src/analysis/runState.integration.test.ts",
|
||||
"test:vacuum-integration": "node --import tsx --test src/health/vacuumBacklog.integration.test.ts",
|
||||
"test:ingest-integration": "timeout 300s node --import tsx --test src/db/packetBatch.integration.test.ts src/db/recentRead.integration.test.ts && timeout 90s node --import tsx --test src/ram/ram.integration.test.ts && timeout 60s node --import tsx --test src/ws/publicAuthorization.integration.test.ts",
|
||||
"test:ram-operational": "timeout 300s node ../scripts/test-ram-redis-acl.mjs",
|
||||
"typecheck": "tsc --noEmit",
|
||||
|
||||
@@ -7,6 +7,8 @@ import {
|
||||
dataLifecycleConfigurationStatus,
|
||||
} from '../db/dataLifecycle.js';
|
||||
import { INGEST_HEALTH_SQL, PATH_HASH_HEALTH_SQL } from './packetDiagnostics.js';
|
||||
import { DATABASE_MAINTENANCE_SQL } from './vacuumBacklog.js';
|
||||
export { DATABASE_MAINTENANCE_SQL } from './vacuumBacklog.js';
|
||||
import { linkWorkerHeartbeatCollector, readLinkWorkerHeartbeatAge } from './workerHeartbeat.js';
|
||||
import {
|
||||
loadVerifiedRestoreReceipt,
|
||||
@@ -731,24 +733,6 @@ async function getOperationalChecksCached(): Promise<{ rows: OperationalCheckRow
|
||||
return { rows: await operationalChecksInflight! };
|
||||
}
|
||||
|
||||
/**
|
||||
* Count tables whose dead tuples indicate a vacuum backlog. The absolute
|
||||
* branch is intentional: TimescaleDB chunks can report very few live tuples
|
||||
* after deletes, so the relative n_live_tup threshold alone misses them.
|
||||
*/
|
||||
export const DATABASE_MAINTENANCE_SQL = `SELECT
|
||||
pg_database_size(current_database())::text AS database_size_bytes,
|
||||
COALESCE(SUM(n_dead_tup), 0)::text AS dead_rows,
|
||||
MIN(last_autovacuum)::text AS oldest_vacuum_at,
|
||||
COUNT(*) FILTER (
|
||||
WHERE (
|
||||
n_live_tup > 10000
|
||||
AND n_dead_tup > GREATEST(10000, n_live_tup * 0.1)
|
||||
)
|
||||
OR n_dead_tup > 50000
|
||||
)::text AS tables_needing_vacuum
|
||||
FROM pg_stat_user_tables`;
|
||||
|
||||
export async function getWorkerHealthOverview() {
|
||||
// Compute system stats once — cpuUsagePct() diffs against lastCpuSample,
|
||||
// so calling it twice in one request gives a garbage near-zero second reading.
|
||||
@@ -976,7 +960,7 @@ export function databaseVacuumBacklogProblem(tablesNeedingVacuum: number): Healt
|
||||
return {
|
||||
code: 'database_vacuum_backlog',
|
||||
severity: tablesNeedingVacuum >= 3 ? 'critical' : 'warning',
|
||||
message: `${Math.floor(tablesNeedingVacuum)} database table(s) exceed the dead-row vacuum threshold`,
|
||||
message: `${Math.floor(tablesNeedingVacuum)} database table(s) exceed the configured vacuum trigger (10,000 dead-row minimum)`,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
DATABASE_MAINTENANCE_SQL,
|
||||
databaseVacuumBacklogProblem,
|
||||
privacyRematerializationBacklogProblem,
|
||||
summarizeAuthoritativeHealth,
|
||||
@@ -40,11 +39,12 @@ test('privacy rematerialization backlog ignores active work and classifies stale
|
||||
assert.equal(privacyRematerializationBacklogProblem({ ...state, pending: 1, oldestPendingAgeSeconds: 21_601 })?.severity, 'critical');
|
||||
});
|
||||
|
||||
test('vacuum health counts dead-only tables and preserves severity thresholds', () => {
|
||||
assert.match(DATABASE_MAINTENANCE_SQL, /n_live_tup > 10000/);
|
||||
assert.match(DATABASE_MAINTENANCE_SQL, /OR n_dead_tup > 50000/);
|
||||
test('vacuum backlog counts preserve warning and critical severity thresholds', () => {
|
||||
assert.equal(databaseVacuumBacklogProblem(0), null);
|
||||
assert.equal(databaseVacuumBacklogProblem(1)?.severity, 'warning');
|
||||
assert.equal(databaseVacuumBacklogProblem(2)?.severity, 'warning');
|
||||
assert.equal(databaseVacuumBacklogProblem(3)?.severity, 'critical');
|
||||
assert.equal(databaseVacuumBacklogProblem(Number.NaN), null);
|
||||
assert.equal(databaseVacuumBacklogProblem(-1), null);
|
||||
assert.equal(databaseVacuumBacklogProblem(Number.POSITIVE_INFINITY), null);
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { after, before, test } from 'node:test';
|
||||
import {
|
||||
DATABASE_MAINTENANCE_SQL,
|
||||
DATABASE_VACUUM_THRESHOLD_SQL,
|
||||
} from './vacuumBacklog.js';
|
||||
|
||||
// CI uses its existing disposable PostgreSQL service. Local verification can
|
||||
// use a worktree-local PGlite installation without any server or credentials.
|
||||
// Every server query is read-only; no schema, migrations or fixtures are written.
|
||||
const connectionString = process.env['TEST_VACUUM_DATABASE_URL'];
|
||||
const moduleUrl = process.env['TEST_VACUUM_PGLITE_MODULE'];
|
||||
let db: { query(sql: string, values?: unknown[]): Promise<{ rows: Record<string, unknown>[] }> };
|
||||
let close: () => Promise<void>;
|
||||
|
||||
before(async () => {
|
||||
if (moduleUrl) {
|
||||
const { PGlite } = await import(moduleUrl);
|
||||
const instance = await PGlite.create();
|
||||
db = instance;
|
||||
close = () => instance.close();
|
||||
} else {
|
||||
assert.ok(connectionString, 'Set TEST_VACUUM_DATABASE_URL or TEST_VACUUM_PGLITE_MODULE');
|
||||
const { Client } = await import('pg');
|
||||
const client = new Client({
|
||||
connectionString,
|
||||
application_name: 'ukmesh-vacuum-policy-tests',
|
||||
options: '-c default_transaction_read_only=on -c statement_timeout=10000',
|
||||
});
|
||||
await client.connect();
|
||||
db = client;
|
||||
close = () => client.end();
|
||||
}
|
||||
});
|
||||
|
||||
after(async () => { if (close) await close(); });
|
||||
|
||||
type Scenario = {
|
||||
name: string;
|
||||
rows: number;
|
||||
dead: number;
|
||||
options?: string[];
|
||||
enabled?: boolean;
|
||||
trackCounts?: boolean;
|
||||
base?: number;
|
||||
scale?: number;
|
||||
threshold: number;
|
||||
backlog: boolean;
|
||||
};
|
||||
|
||||
const scenarios: Scenario[] = [
|
||||
{ name: 'incident: multibyte churn below its trigger', rows: 2_469_879, dead: 285_751, threshold: 494_025.8, backlog: false },
|
||||
{ name: 'incident: hourly stats below its trigger', rows: 909_590, dead: 108_462, threshold: 181_968, backlog: false },
|
||||
{ name: 'incident: observer sightings below their trigger', rows: 233_931, dead: 35_693, threshold: 46_836.2, backlog: false },
|
||||
{ name: 'incident: share deliveries below their trigger', rows: 234_784, dead: 27_720, threshold: 47_006.8, backlog: false },
|
||||
{ name: 'incident: public feed below its trigger', rows: 98_132, dead: 18_527, threshold: 19_676.4, backlog: false },
|
||||
{ name: 'large table crossing its trigger still alerts', rows: 2_469_879, dead: 494_026, threshold: 494_025.8, backlog: true },
|
||||
{ name: 'exact trigger is not exceeded', rows: 100_000, dead: 20_050, threshold: 20_050, backlog: false },
|
||||
{ name: 'one dead row above trigger alerts', rows: 100_000, dead: 20_051, threshold: 20_050, backlog: true },
|
||||
{ name: 'table scale overrides server default', rows: 1_000_000, dead: 60_000, options: ['autovacuum_vacuum_scale_factor=0.05'], threshold: 50_050, backlog: true },
|
||||
{ name: 'table base overrides server default', rows: 100_000, dead: 22_001, options: ['autovacuum_vacuum_threshold=2000'], threshold: 22_000, backlog: true },
|
||||
{ name: 'zero-valued overrides are honored', rows: 1_000_000, dead: 10_001, options: ['autovacuum_vacuum_threshold=0', 'autovacuum_vacuum_scale_factor=0'], threshold: 10_000, backlog: true },
|
||||
{ name: 'both overrides win over changed defaults', rows: 1_000_000, dead: 51_001, base: 10_000, scale: 0.5, options: ['autovacuum_vacuum_threshold=1000', 'autovacuum_vacuum_scale_factor=0.05'], threshold: 51_000, backlog: true },
|
||||
{ name: 'server threshold and scale are used as fallback', rows: 100_000, dead: 31_001, base: 1000, scale: 0.3, threshold: 31_000, backlog: true },
|
||||
{ name: 'unrelated table options do not change trigger', rows: 100_000, dead: 20_051, options: ['autovacuum_analyze_scale_factor=0.01', 'fillfactor=80'], threshold: 20_050, backlog: true },
|
||||
{ name: 'dead-only tables are not excluded', rows: 0, dead: 50_001, threshold: 10_000, backlog: true },
|
||||
{ name: 'unknown row estimate is treated as zero', rows: -1, dead: 10_001, threshold: 10_000, backlog: true },
|
||||
{ name: 'noise floor excludes small transient churn', rows: 100, dead: 10_000, threshold: 10_000, backlog: false },
|
||||
{ name: 'disabled table vacuum cannot hide dead rows', rows: 1_000_000, dead: 10_001, options: ['autovacuum_enabled=false'], threshold: 10_000, backlog: true },
|
||||
{ name: 'disabled server vacuum cannot be reenabled by table', rows: 1_000_000, dead: 10_001, enabled: false, options: ['autovacuum_enabled=true'], threshold: 10_000, backlog: true },
|
||||
{ name: 'disabled statistics collection uses conservative floor', rows: 1_000_000, dead: 10_001, trackCounts: false, threshold: 10_000, backlog: true },
|
||||
];
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
test(scenario.name, async () => {
|
||||
const result = await db.query(`WITH c AS (
|
||||
SELECT $1::real AS reltuples, $2::text[] AS reloptions
|
||||
), settings AS (
|
||||
SELECT $3::boolean AS autovacuum_enabled, $4::boolean AS track_counts,
|
||||
$5::double precision AS vacuum_threshold, $6::double precision AS vacuum_scale_factor
|
||||
), threshold AS (
|
||||
SELECT ${DATABASE_VACUUM_THRESHOLD_SQL} AS value FROM c CROSS JOIN settings
|
||||
) SELECT value, $7::bigint > value AS needs_vacuum FROM threshold`, [
|
||||
scenario.rows, scenario.options ?? null, scenario.enabled ?? true,
|
||||
scenario.trackCounts ?? true, scenario.base ?? 50, scenario.scale ?? 0.2,
|
||||
scenario.dead,
|
||||
]);
|
||||
assert.ok(Math.abs(Number(result.rows[0]?.value) - scenario.threshold) < 0.000001);
|
||||
assert.equal(result.rows[0]?.needs_vacuum, scenario.backlog);
|
||||
});
|
||||
}
|
||||
|
||||
test('the complete health catalog query executes inside a read-only transaction', async () => {
|
||||
await db.query('BEGIN READ ONLY');
|
||||
try {
|
||||
assert.equal((await db.query("SELECT current_setting('transaction_read_only') AS mode")).rows[0]?.mode, 'on');
|
||||
const result = await db.query(DATABASE_MAINTENANCE_SQL);
|
||||
assert.equal(result.rows.length, 1);
|
||||
assert.ok(Number(result.rows[0]?.database_size_bytes) > 0);
|
||||
assert.ok(Number(result.rows[0]?.dead_rows) >= 0);
|
||||
assert.ok(Number(result.rows[0]?.tables_needing_vacuum) >= 0);
|
||||
} finally {
|
||||
await db.query('ROLLBACK');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
/** PostgreSQL 16's update/delete vacuum trigger, with a health noise floor.
|
||||
*
|
||||
* Use pg_class.reltuples, not n_live_tup: it is the estimate the autovacuum
|
||||
* scheduler uses. Table storage parameters override the server defaults.
|
||||
* Disabled autovacuum (or statistics collection) must not hide a backlog.
|
||||
* Insert/freeze-triggered vacuums are separate from this dead-row check.
|
||||
*/
|
||||
export const DATABASE_VACUUM_SETTINGS_SQL = `SELECT
|
||||
current_setting('autovacuum')::boolean AS autovacuum_enabled,
|
||||
current_setting('track_counts')::boolean AS track_counts,
|
||||
current_setting('autovacuum_vacuum_threshold')::double precision AS vacuum_threshold,
|
||||
current_setting('autovacuum_vacuum_scale_factor')::double precision AS vacuum_scale_factor`;
|
||||
|
||||
// c and settings are supplied by both the catalog query and SQL test fixtures.
|
||||
export const DATABASE_VACUUM_THRESHOLD_SQL = `GREATEST(
|
||||
10000::double precision,
|
||||
CASE WHEN settings.autovacuum_enabled AND settings.track_counts
|
||||
AND COALESCE((SELECT option_value::boolean
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_enabled'), TRUE)
|
||||
THEN COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_threshold'), settings.vacuum_threshold)
|
||||
+ COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_scale_factor'), settings.vacuum_scale_factor)
|
||||
* GREATEST(c.reltuples, 0)
|
||||
ELSE 0 END
|
||||
)`;
|
||||
|
||||
/** Catalog-only: never scans application rows or invokes maintenance. */
|
||||
export const DATABASE_VACUUM_TABLES_SQL = `WITH settings AS (
|
||||
${DATABASE_VACUUM_SETTINGS_SQL}
|
||||
)
|
||||
SELECT s.*, c.reltuples, c.reloptions,
|
||||
${DATABASE_VACUUM_THRESHOLD_SQL} AS health_dead_row_threshold
|
||||
FROM pg_catalog.pg_stat_user_tables s
|
||||
JOIN pg_catalog.pg_class c ON c.oid = s.relid
|
||||
CROSS JOIN settings`;
|
||||
|
||||
export const DATABASE_MAINTENANCE_SQL = `SELECT
|
||||
pg_database_size(current_database())::text AS database_size_bytes,
|
||||
COALESCE(SUM(n_dead_tup), 0)::text AS dead_rows,
|
||||
MIN(last_autovacuum)::text AS oldest_vacuum_at,
|
||||
COUNT(*) FILTER (WHERE n_dead_tup > health_dead_row_threshold)::text AS tables_needing_vacuum
|
||||
FROM (${DATABASE_VACUUM_TABLES_SQL}) table_health`;
|
||||
@@ -0,0 +1,53 @@
|
||||
import { DATABASE_MAINTENANCE_SQL, DATABASE_VACUUM_TABLES_SQL } from '../health/vacuumBacklog.js';
|
||||
|
||||
// Emit psql input so the fleet can use its existing SSH/docker/psql access.
|
||||
// No connection, credential loading, application bootstrap or migration runner.
|
||||
process.stdout.write(`\\set ON_ERROR_STOP on
|
||||
\\pset pager off
|
||||
BEGIN READ ONLY;
|
||||
SET LOCAL statement_timeout = '10s';
|
||||
SET LOCAL lock_timeout = '2s';
|
||||
SET LOCAL idle_in_transaction_session_timeout = '30s';
|
||||
SELECT now() AS measured_at, current_database(), current_user,
|
||||
current_setting('server_version') AS server_version,
|
||||
current_setting('transaction_read_only') AS read_only;
|
||||
SELECT name, setting, unit, source FROM pg_settings
|
||||
WHERE name LIKE 'autovacuum%'
|
||||
OR name IN ('track_counts', 'vacuum_cost_delay', 'vacuum_cost_limit',
|
||||
'maintenance_work_mem', 'max_prepared_transactions', 'max_worker_processes')
|
||||
ORDER BY name;
|
||||
${DATABASE_MAINTENANCE_SQL};
|
||||
SELECT schemaname, relname, reltuples, n_live_tup, n_dead_tup,
|
||||
health_dead_row_threshold, n_dead_tup > health_dead_row_threshold AS needs_vacuum,
|
||||
n_tup_ins, n_tup_upd, n_tup_hot_upd, n_tup_del,
|
||||
last_vacuum, last_autovacuum, last_analyze, last_autoanalyze,
|
||||
vacuum_count, autovacuum_count, reloptions,
|
||||
pg_table_size(relid) AS table_bytes, pg_indexes_size(relid) AS index_bytes,
|
||||
pg_total_relation_size(relid) AS total_bytes
|
||||
FROM (${DATABASE_VACUUM_TABLES_SQL}) table_health
|
||||
ORDER BY n_dead_tup DESC LIMIT 30;
|
||||
-- Intentionally omit SQL text: activity queries can contain secrets/user data.
|
||||
SELECT pid, datname, backend_type, application_name, state,
|
||||
now() - xact_start AS transaction_age, now() - query_start AS query_age,
|
||||
backend_xid, backend_xmin, wait_event_type, wait_event,
|
||||
pg_blocking_pids(pid) AS blockers
|
||||
FROM pg_stat_activity
|
||||
WHERE pid <> pg_backend_pid() AND (
|
||||
xact_start < now() - interval '1 minute'
|
||||
OR backend_xmin IS NOT NULL OR backend_type = 'autovacuum worker')
|
||||
ORDER BY xact_start NULLS LAST;
|
||||
-- GIDs are also omitted; age and database suffice to identify pinned horizons.
|
||||
SELECT transaction, prepared, now() - prepared AS prepared_age, owner, database
|
||||
FROM pg_prepared_xacts;
|
||||
SELECT slot_name, slot_type, database, active, xmin, catalog_xmin,
|
||||
restart_lsn, confirmed_flush_lsn FROM pg_replication_slots;
|
||||
SELECT pid, datname, relid::regclass AS relation, phase, heap_blks_total,
|
||||
heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count
|
||||
FROM pg_stat_progress_vacuum;
|
||||
SELECT datname, stats_reset, deadlocks, xact_commit, xact_rollback
|
||||
FROM pg_stat_database WHERE datname = current_database();
|
||||
SELECT age(datfrozenxid) AS database_xid_age,
|
||||
current_setting('autovacuum_freeze_max_age') AS freeze_max_age
|
||||
FROM pg_database WHERE datname = current_database();
|
||||
COMMIT;
|
||||
`);
|
||||
@@ -0,0 +1,103 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\pset pager off
|
||||
BEGIN READ ONLY;
|
||||
SET LOCAL statement_timeout = '10s';
|
||||
SET LOCAL lock_timeout = '2s';
|
||||
SET LOCAL idle_in_transaction_session_timeout = '30s';
|
||||
SELECT now() AS measured_at, current_database(), current_user,
|
||||
current_setting('server_version') AS server_version,
|
||||
current_setting('transaction_read_only') AS read_only;
|
||||
SELECT name, setting, unit, source FROM pg_settings
|
||||
WHERE name LIKE 'autovacuum%'
|
||||
OR name IN ('track_counts', 'vacuum_cost_delay', 'vacuum_cost_limit',
|
||||
'maintenance_work_mem', 'max_prepared_transactions', 'max_worker_processes')
|
||||
ORDER BY name;
|
||||
SELECT
|
||||
pg_database_size(current_database())::text AS database_size_bytes,
|
||||
COALESCE(SUM(n_dead_tup), 0)::text AS dead_rows,
|
||||
MIN(last_autovacuum)::text AS oldest_vacuum_at,
|
||||
COUNT(*) FILTER (WHERE n_dead_tup > health_dead_row_threshold)::text AS tables_needing_vacuum
|
||||
FROM (WITH settings AS (
|
||||
SELECT
|
||||
current_setting('autovacuum')::boolean AS autovacuum_enabled,
|
||||
current_setting('track_counts')::boolean AS track_counts,
|
||||
current_setting('autovacuum_vacuum_threshold')::double precision AS vacuum_threshold,
|
||||
current_setting('autovacuum_vacuum_scale_factor')::double precision AS vacuum_scale_factor
|
||||
)
|
||||
SELECT s.*, c.reltuples, c.reloptions,
|
||||
GREATEST(
|
||||
10000::double precision,
|
||||
CASE WHEN settings.autovacuum_enabled AND settings.track_counts
|
||||
AND COALESCE((SELECT option_value::boolean
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_enabled'), TRUE)
|
||||
THEN COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_threshold'), settings.vacuum_threshold)
|
||||
+ COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_scale_factor'), settings.vacuum_scale_factor)
|
||||
* GREATEST(c.reltuples, 0)
|
||||
ELSE 0 END
|
||||
) AS health_dead_row_threshold
|
||||
FROM pg_catalog.pg_stat_user_tables s
|
||||
JOIN pg_catalog.pg_class c ON c.oid = s.relid
|
||||
CROSS JOIN settings) table_health;
|
||||
SELECT schemaname, relname, reltuples, n_live_tup, n_dead_tup,
|
||||
health_dead_row_threshold, n_dead_tup > health_dead_row_threshold AS needs_vacuum,
|
||||
n_tup_ins, n_tup_upd, n_tup_hot_upd, n_tup_del,
|
||||
last_vacuum, last_autovacuum, last_analyze, last_autoanalyze,
|
||||
vacuum_count, autovacuum_count, reloptions,
|
||||
pg_table_size(relid) AS table_bytes, pg_indexes_size(relid) AS index_bytes,
|
||||
pg_total_relation_size(relid) AS total_bytes
|
||||
FROM (WITH settings AS (
|
||||
SELECT
|
||||
current_setting('autovacuum')::boolean AS autovacuum_enabled,
|
||||
current_setting('track_counts')::boolean AS track_counts,
|
||||
current_setting('autovacuum_vacuum_threshold')::double precision AS vacuum_threshold,
|
||||
current_setting('autovacuum_vacuum_scale_factor')::double precision AS vacuum_scale_factor
|
||||
)
|
||||
SELECT s.*, c.reltuples, c.reloptions,
|
||||
GREATEST(
|
||||
10000::double precision,
|
||||
CASE WHEN settings.autovacuum_enabled AND settings.track_counts
|
||||
AND COALESCE((SELECT option_value::boolean
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_enabled'), TRUE)
|
||||
THEN COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_threshold'), settings.vacuum_threshold)
|
||||
+ COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_scale_factor'), settings.vacuum_scale_factor)
|
||||
* GREATEST(c.reltuples, 0)
|
||||
ELSE 0 END
|
||||
) AS health_dead_row_threshold
|
||||
FROM pg_catalog.pg_stat_user_tables s
|
||||
JOIN pg_catalog.pg_class c ON c.oid = s.relid
|
||||
CROSS JOIN settings) table_health
|
||||
ORDER BY n_dead_tup DESC LIMIT 30;
|
||||
-- Intentionally omit SQL text: activity queries can contain secrets/user data.
|
||||
SELECT pid, datname, backend_type, application_name, state,
|
||||
now() - xact_start AS transaction_age, now() - query_start AS query_age,
|
||||
backend_xid, backend_xmin, wait_event_type, wait_event,
|
||||
pg_blocking_pids(pid) AS blockers
|
||||
FROM pg_stat_activity
|
||||
WHERE pid <> pg_backend_pid() AND (
|
||||
xact_start < now() - interval '1 minute'
|
||||
OR backend_xmin IS NOT NULL OR backend_type = 'autovacuum worker')
|
||||
ORDER BY xact_start NULLS LAST;
|
||||
-- GIDs are also omitted; age and database suffice to identify pinned horizons.
|
||||
SELECT transaction, prepared, now() - prepared AS prepared_age, owner, database
|
||||
FROM pg_prepared_xacts;
|
||||
SELECT slot_name, slot_type, database, active, xmin, catalog_xmin,
|
||||
restart_lsn, confirmed_flush_lsn FROM pg_replication_slots;
|
||||
SELECT pid, datname, relid::regclass AS relation, phase, heap_blks_total,
|
||||
heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count
|
||||
FROM pg_stat_progress_vacuum;
|
||||
SELECT datname, stats_reset, deadlocks, xact_commit, xact_rollback
|
||||
FROM pg_stat_database WHERE datname = current_database();
|
||||
SELECT age(datfrozenxid) AS database_xid_age,
|
||||
current_setting('autovacuum_freeze_max_age') AS freeze_max_age
|
||||
FROM pg_database WHERE datname = current_database();
|
||||
COMMIT;
|
||||
@@ -0,0 +1,102 @@
|
||||
Pager usage is off.
|
||||
BEGIN
|
||||
SET
|
||||
SET
|
||||
SET
|
||||
measured_at | current_database | current_user | server_version | read_only
|
||||
-------------------------------+------------------+--------------+----------------+-----------
|
||||
2026-10-22 14:25:27.529259+00 | meshcore | meshcore | 16.11 | on
|
||||
(1 row)
|
||||
|
||||
name | setting | unit | source
|
||||
---------------------------------------+-----------+------+--------------------
|
||||
autovacuum | on | | default
|
||||
autovacuum_analyze_scale_factor | 0.1 | | default
|
||||
autovacuum_analyze_threshold | 50 | | default
|
||||
autovacuum_freeze_max_age | 200000000 | | default
|
||||
autovacuum_max_workers | 3 | | configuration file
|
||||
autovacuum_multixact_freeze_max_age | 400000000 | | default
|
||||
autovacuum_naptime | 10 | s | configuration file
|
||||
autovacuum_vacuum_cost_delay | 2 | ms | default
|
||||
autovacuum_vacuum_cost_limit | -1 | | default
|
||||
autovacuum_vacuum_insert_scale_factor | 0.2 | | default
|
||||
autovacuum_vacuum_insert_threshold | 1000 | | default
|
||||
autovacuum_vacuum_scale_factor | 0.2 | | default
|
||||
autovacuum_vacuum_threshold | 50 | | default
|
||||
autovacuum_work_mem | -1 | kB | default
|
||||
maintenance_work_mem | 251903 | kB | configuration file
|
||||
max_prepared_transactions | 0 | | default
|
||||
max_worker_processes | 24 | | command line
|
||||
track_counts | on | | default
|
||||
vacuum_cost_delay | 0 | ms | default
|
||||
vacuum_cost_limit | 200 | | default
|
||||
(20 rows)
|
||||
|
||||
database_size_bytes | dead_rows | oldest_vacuum_at | tables_needing_vacuum
|
||||
---------------------+-----------+-------------------------------+-----------------------
|
||||
17721029091 | 507348 | 2026-10-20 15:41:37.193334+00 | 0
|
||||
(1 row)
|
||||
|
||||
schemaname | relname | reltuples | n_live_tup | n_dead_tup | health_dead_row_threshold | needs_vacuum | n_tup_ins | n_tup_upd | n_tup_hot_upd | n_tup_del | last_vacuum | last_autovacuum | last_analyze | last_autoanalyze | vacuum_count | autovacuum_count | reloptions | table_bytes | index_bytes | total_bytes
|
||||
-----------------------+-----------------------------------------+--------------+------------+------------+---------------------------+--------------+-----------+-----------+---------------+-----------+-------------+-------------------------------+--------------+-------------------------------+--------------+------------------+--------------------------------------------------------------------------+-------------+-------------+-------------
|
||||
public | multibyte_path_facts | 2.469879e+06 | 2473913 | 285751 | 494025.80000000005 | f | 86321 | 2860342 | 50 | 0 | | 2026-10-22 06:07:27.164047+00 | | 2026-10-22 12:05:03.200668+00 | 0 | 5 | | 1801912320 | 400474112 | 2202386432
|
||||
public | packet_hourly_stats | 909590 | 914989 | 108459 | 181968 | f | 45095 | 257250 | 254193 | 0 | | | | 2026-10-22 08:10:50.694911+00 | 0 | 0 | | 144916480 | 81289216 | 226205696
|
||||
public | observer_region_packet_sightings | 233931 | 238022 | 35785 | 46836.200000000004 | f | 62512 | 90869 | 0 | 20668 | | 2026-10-21 16:55:15.159896+00 | | 2026-10-22 11:29:06.44+00 | 0 | 2 | | 34660352 | 276357120 | 311017472
|
||||
public | owner_packet_share_deliveries | 234784 | 241414 | 27746 | 47006.8 | f | 21185 | 42430 | 0 | 0 | | 2026-10-20 20:28:26.808679+00 | | 2026-10-21 18:20:55.586581+00 | 0 | 1 | | 307871744 | 107323392 | 415195136
|
||||
public | public_feed_deliveries | 98132 | 99444 | 18531 | 19676.4 | f | 4475 | 8964 | 0 | 0 | | | | 2026-10-21 19:38:46.719846+00 | 0 | 0 | | 96845824 | 32071680 | 128917504
|
||||
public | node_identity_links | 166632 | 166695 | 5834 | 10000 | f | 31658 | 0 | 0 | 31354 | | 2026-10-22 03:15:13.289109+00 | | 2026-10-22 04:00:23.240615+00 | 0 | 3 | {autovacuum_vacuum_scale_factor=0.05} | 40845312 | 80838656 | 121683968
|
||||
public | node_links | 131925 | 131948 | 5775 | 10000 | f | 305 | 97308 | 11 | 0 | | 2026-10-22 11:33:00.218495+00 | | 2026-10-22 07:27:07.253258+00 | 0 | 12 | {autovacuum_vacuum_scale_factor=0.05} | 113221632 | 44351488 | 157573120
|
||||
public | nodes | 13764 | 13765 | 4579 | 10000 | f | 141 | 1097433 | 656596 | 9417 | | 2026-10-22 13:57:27.133178+00 | | 2026-10-22 13:57:16.321617+00 | 0 | 129 | {autovacuum_vacuum_scale_factor=0.5,autovacuum_analyze_scale_factor=0.5} | 73490432 | 100376576 | 173867008
|
||||
public | path_motif_priors | 110714 | 110774 | 4036 | 10000 | f | 14581 | 169028 | 123468 | 9799 | | 2026-10-22 13:05:04.967151+00 | | 2026-10-22 12:05:49.07874+00 | 0 | 23 | {autovacuum_vacuum_scale_factor=0.05} | 41467904 | 402554880 | 444022784
|
||||
public | path_transition_priors | 19592 | 19587 | 2632 | 10000 | f | 2348 | 96970 | 90731 | 2000 | | 2026-10-22 11:05:36.739611+00 | | 2026-10-22 14:05:01.072423+00 | 0 | 11 | | 8265728 | 31907840 | 40173568
|
||||
public | node_network_sightings | 23665 | 23675 | 2513 | 10000 | f | 297 | 48060 | 47903 | 10636 | | 2026-10-22 08:35:46.860157+00 | | 2026-10-22 12:38:19.111234+00 | 0 | 4 | | 5611520 | 4587520 | 10199040
|
||||
public | path_position_prefix_priors | 6697 | 6727 | 1132 | 10000 | f | 1518 | 26228 | 23965 | 400 | | 2026-10-22 05:05:49.230475+00 | | 2026-10-22 14:05:01.193656+00 | 0 | 9 | | 1695744 | 2269184 | 3964928
|
||||
public | node_identity_sightings | 23574 | 23574 | 1126 | 10000 | f | 8736 | 0 | 0 | 19051 | | 2026-10-22 06:45:07.676025+00 | | 2026-10-22 08:00:08.156425+00 | 0 | 3 | | 4784128 | 18186240 | 22970368
|
||||
public | analysis_runs | 29623 | 830 | 843 | 10000 | f | 830 | 1662 | 826 | 0 | | | | | 0 | 0 | | 21733376 | 9330688 | 31064064
|
||||
public | path_prefix_priors | 41538 | 41514 | 805 | 10000 | f | 2733 | 37211 | 33117 | 3285 | | 2026-10-22 13:05:54.266772+00 | | 2026-10-22 11:05:37.423802+00 | 0 | 10 | {autovacuum_vacuum_scale_factor=0.05} | 9248768 | 58679296 | 67928064
|
||||
public | path_corridor_priors | 3985 | 3956 | 332 | 10000 | f | 786 | 2028 | 1638 | 346 | | 2026-10-21 20:05:20.198002+00 | | 2026-10-22 12:05:19.336969+00 | 0 | 1 | | 5849088 | 28835840 | 34684928
|
||||
public | worker_health_snapshots | 8052 | 8052 | 316 | 10000 | f | 952 | 0 | 0 | 952 | | 2026-10-20 15:41:37.193334+00 | | 2026-10-20 20:17:13.987902+00 | 0 | 1 | | 1122304 | 19775488 | 20897792
|
||||
public | path_position_transition_priors | 2944 | 2928 | 280 | 10000 | f | 470 | 3902 | 3602 | 172 | | 2026-10-22 01:04:47.389715+00 | | 2026-10-22 10:05:28.635468+00 | 0 | 1 | | 1835008 | 5767168 | 7602176
|
||||
public | operational_check_results | 29171 | 30084 | 222 | 10000 | f | 4800 | 0 | 0 | 1589 | | 2026-10-20 19:32:33.959082+00 | | 2026-10-22 03:00:23.636254+00 | 0 | 1 | | 3031040 | 6979584 | 10010624
|
||||
public | spam_suspects | 3891 | 1 | 216 | 10000 | f | 1 | 216 | 0 | 0 | | | | | 0 | 0 | | 2793472 | 13967360 | 16760832
|
||||
public | path_slow_resolutions | 216288 | 12399 | 94 | 43307.600000000006 | f | 12399 | 94 | 0 | 0 | | | | | 0 | 0 | | 398163968 | 38772736 | 436936704
|
||||
public | spam_message_members | 1178 | 1178 | 65 | 10000 | f | 4355 | 0 | 0 | 4326 | | 2026-10-22 14:01:16.225221+00 | | 2026-10-22 14:01:16.265405+00 | 0 | 16 | | 2342912 | 679936 | 3022848
|
||||
_timescaledb_internal | bgw_job_stat | 9 | 0 | 39 | 10000 | f | 0 | 39 | 39 | 0 | | | | | 0 | 0 | | 40960 | 16384 | 57344
|
||||
public | packet_daily_stats | 197 | 2 | 33 | 10000 | f | 2 | 33 | 0 | 0 | | | | | 0 | 0 | | 65536 | 49152 | 114688
|
||||
public | worker_health_current | 4 | 4 | 32 | 10000 | f | 0 | 952 | 952 | 0 | | 2026-10-20 21:42:23.477984+00 | | 2026-10-20 21:57:22.934889+00 | 0 | 17 | | 114688 | 16384 | 131072
|
||||
public | analysis_workload_state | 7 | 7 | 28 | 10000 | f | 0 | 1664 | 0 | 0 | | 2026-10-22 13:25:46.479945+00 | | 2026-10-22 13:30:39.438276+00 | 0 | 32 | | 131072 | 32768 | 163840
|
||||
public | spam_message_incidents | 73 | 73 | 25 | 10000 | f | 3 | 421 | 413 | 0 | | | | 2026-10-22 12:50:05.645858+00 | 0 | 0 | | 901120 | 49152 | 950272
|
||||
public | packet_visibility_materialization_state | 1 | 0 | 18 | 10000 | f | 0 | 18 | 18 | 0 | | | | | 0 | 0 | | 40960 | 16384 | 57344
|
||||
public | public_visibility_state | 1 | 0 | 18 | 10000 | f | 0 | 18 | 18 | 0 | | | | | 0 | 0 | | 40960 | 16384 | 57344
|
||||
public | path_model_calibration | 4 | 4 | 17 | 10000 | f | 0 | 118 | 118 | 0 | | | | 2026-10-22 07:04:59.496947+00 | 0 | 0 | | 57344 | 16384 | 73728
|
||||
(30 rows)
|
||||
|
||||
pid | datname | backend_type | application_name | state | transaction_age | query_age | backend_xid | backend_xmin | wait_event_type | wait_event | blockers
|
||||
---------+----------+----------------+------------------+--------+-----------------+-----------------+-------------+--------------+-----------------+------------+----------
|
||||
4164893 | meshcore | client backend | | active | 00:00:10.912921 | 00:00:10.539898 | | 2150156089 | | | {}
|
||||
4164896 | meshcore | client backend | meshcore-backend | active | 00:00:03.134097 | 00:00:03.122643 | | 2150156101 | | | {}
|
||||
(2 rows)
|
||||
|
||||
transaction | prepared | prepared_age | owner | database
|
||||
-------------+----------+--------------+-------+----------
|
||||
(0 rows)
|
||||
|
||||
slot_name | slot_type | database | active | xmin | catalog_xmin | restart_lsn | confirmed_flush_lsn
|
||||
-----------+-----------+----------+--------+------+--------------+-------------+---------------------
|
||||
(0 rows)
|
||||
|
||||
pid | datname | relation | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count
|
||||
-----+---------+----------+-------+-----------------+-------------------+--------------------+--------------------
|
||||
(0 rows)
|
||||
|
||||
datname | stats_reset | deadlocks | xact_commit | xact_rollback
|
||||
----------+-------------+-----------+-------------+---------------
|
||||
meshcore | | 0 | 3845184 | 683
|
||||
(1 row)
|
||||
|
||||
database_xid_age | freeze_max_age
|
||||
------------------+----------------
|
||||
132622753 | 200000000
|
||||
(1 row)
|
||||
|
||||
COMMIT
|
||||
@@ -0,0 +1,66 @@
|
||||
BEGIN READ ONLY;
|
||||
SET LOCAL statement_timeout='10s';
|
||||
SELECT COUNT(*) FILTER (WHERE (n_live_tup>10000 AND n_dead_tup>GREATEST(10000,n_live_tup*0.1)) OR n_dead_tup>50000) AS previous_tables_needing_vacuum FROM pg_stat_user_tables;
|
||||
SELECT
|
||||
pg_database_size(current_database())::text AS database_size_bytes,
|
||||
COALESCE(SUM(n_dead_tup), 0)::text AS dead_rows,
|
||||
MIN(last_autovacuum)::text AS oldest_vacuum_at,
|
||||
COUNT(*) FILTER (WHERE n_dead_tup > health_dead_row_threshold)::text AS tables_needing_vacuum
|
||||
FROM (WITH settings AS (
|
||||
SELECT
|
||||
current_setting('autovacuum')::boolean AS autovacuum_enabled,
|
||||
current_setting('track_counts')::boolean AS track_counts,
|
||||
current_setting('autovacuum_vacuum_threshold')::double precision AS vacuum_threshold,
|
||||
current_setting('autovacuum_vacuum_scale_factor')::double precision AS vacuum_scale_factor
|
||||
)
|
||||
SELECT s.*, c.reltuples, c.reloptions,
|
||||
GREATEST(
|
||||
10000::double precision,
|
||||
CASE WHEN settings.autovacuum_enabled AND settings.track_counts
|
||||
AND COALESCE((SELECT option_value::boolean
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_enabled'), TRUE)
|
||||
THEN COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_threshold'), settings.vacuum_threshold)
|
||||
+ COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_scale_factor'), settings.vacuum_scale_factor)
|
||||
* GREATEST(c.reltuples, 0)
|
||||
ELSE 0 END
|
||||
) AS health_dead_row_threshold
|
||||
FROM pg_catalog.pg_stat_user_tables s
|
||||
JOIN pg_catalog.pg_class c ON c.oid = s.relid
|
||||
CROSS JOIN settings) table_health;
|
||||
EXPLAIN (ANALYZE, BUFFERS, TIMING OFF) SELECT
|
||||
pg_database_size(current_database())::text AS database_size_bytes,
|
||||
COALESCE(SUM(n_dead_tup), 0)::text AS dead_rows,
|
||||
MIN(last_autovacuum)::text AS oldest_vacuum_at,
|
||||
COUNT(*) FILTER (WHERE n_dead_tup > health_dead_row_threshold)::text AS tables_needing_vacuum
|
||||
FROM (WITH settings AS (
|
||||
SELECT
|
||||
current_setting('autovacuum')::boolean AS autovacuum_enabled,
|
||||
current_setting('track_counts')::boolean AS track_counts,
|
||||
current_setting('autovacuum_vacuum_threshold')::double precision AS vacuum_threshold,
|
||||
current_setting('autovacuum_vacuum_scale_factor')::double precision AS vacuum_scale_factor
|
||||
)
|
||||
SELECT s.*, c.reltuples, c.reloptions,
|
||||
GREATEST(
|
||||
10000::double precision,
|
||||
CASE WHEN settings.autovacuum_enabled AND settings.track_counts
|
||||
AND COALESCE((SELECT option_value::boolean
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_enabled'), TRUE)
|
||||
THEN COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_threshold'), settings.vacuum_threshold)
|
||||
+ COALESCE((SELECT option_value::double precision
|
||||
FROM pg_catalog.pg_options_to_table(c.reloptions)
|
||||
WHERE option_name = 'autovacuum_vacuum_scale_factor'), settings.vacuum_scale_factor)
|
||||
* GREATEST(c.reltuples, 0)
|
||||
ELSE 0 END
|
||||
) AS health_dead_row_threshold
|
||||
FROM pg_catalog.pg_stat_user_tables s
|
||||
JOIN pg_catalog.pg_class c ON c.oid = s.relid
|
||||
CROSS JOIN settings) table_health;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,70 @@
|
||||
BEGIN
|
||||
SET
|
||||
previous_tables_needing_vacuum
|
||||
--------------------------------
|
||||
5
|
||||
(1 row)
|
||||
|
||||
database_size_bytes | dead_rows | oldest_vacuum_at | tables_needing_vacuum
|
||||
---------------------+-----------+-------------------------------+-----------------------
|
||||
17721905635 | 503544 | 2026-10-20 15:41:37.193334+00 | 0
|
||||
(1 row)
|
||||
|
||||
QUERY PLAN
|
||||
-------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Aggregate (cost=396.05..396.08 rows=1 width=128) (actual rows=1 loops=1)
|
||||
Buffers: shared hit=363
|
||||
-> Hash Join (cost=329.86..389.43 rows=294 width=360) (actual rows=247 loops=1)
|
||||
Hash Cond: (c_1.oid = c.oid)
|
||||
Buffers: shared hit=361
|
||||
-> HashAggregate (cost=185.64..190.05 rows=294 width=316) (actual rows=247 loops=1)
|
||||
Group Key: c_1.oid, n.nspname
|
||||
Batches: 1 Memory Usage: 157kB
|
||||
Buffers: shared hit=267
|
||||
-> Hash Right Join (cost=137.41..184.17 rows=294 width=132) (actual rows=1096 loops=1)
|
||||
Hash Cond: (i.indrelid = c_1.oid)
|
||||
Buffers: shared hit=267
|
||||
-> Index Only Scan using pg_index_indrelid_index on pg_index i (cost=0.28..39.58 rows=1470 width=4) (actual rows=1470 loops=1)
|
||||
Heap Fetches: 497
|
||||
Buffers: shared hit=171
|
||||
-> Hash (cost=133.46..133.46 rows=294 width=132) (actual rows=247 loops=1)
|
||||
Buckets: 1024 Batches: 1 Memory Usage: 48kB
|
||||
Buffers: shared hit=96
|
||||
-> Hash Join (cost=4.42..133.46 rows=294 width=132) (actual rows=247 loops=1)
|
||||
Hash Cond: (c_1.relnamespace = n.oid)
|
||||
Buffers: shared hit=96
|
||||
-> Seq Scan on pg_class c_1 (cost=0.00..127.48 rows=573 width=72) (actual rows=573 loops=1)
|
||||
Filter: (relkind = ANY ('{r,t,m,p}'::"char"[]))
|
||||
Rows Removed by Filter: 1659
|
||||
Buffers: shared hit=94
|
||||
-> Hash (cost=3.69..3.69 rows=58 width=68) (actual rows=67 loops=1)
|
||||
Buckets: 1024 Batches: 1 Memory Usage: 15kB
|
||||
Buffers: shared hit=2
|
||||
-> Seq Scan on pg_namespace n (cost=0.00..3.69 rows=58 width=68) (actual rows=67 loops=1)
|
||||
Filter: ((nspname <> ALL ('{pg_catalog,information_schema}'::name[])) AND (nspname !~ '^pg_toast'::text))
|
||||
Rows Removed by Filter: 61
|
||||
Buffers: shared hit=2
|
||||
-> Hash (cost=116.32..116.32 rows=2232 width=58) (actual rows=2232 loops=1)
|
||||
Buckets: 4096 Batches: 1 Memory Usage: 123kB
|
||||
Buffers: shared hit=94
|
||||
-> Seq Scan on pg_class c (cost=0.00..116.32 rows=2232 width=58) (actual rows=2232 loops=1)
|
||||
Buffers: shared hit=94
|
||||
SubPlan 1
|
||||
-> Function Scan on pg_options_to_table (cost=0.00..0.05 rows=1 width=1) (actual rows=0 loops=247)
|
||||
Filter: (option_name = 'autovacuum_enabled'::text)
|
||||
Rows Removed by Filter: 0
|
||||
SubPlan 2
|
||||
-> Function Scan on pg_options_to_table pg_options_to_table_1 (cost=0.00..0.05 rows=1 width=8) (actual rows=0 loops=247)
|
||||
Filter: (option_name = 'autovacuum_vacuum_threshold'::text)
|
||||
Rows Removed by Filter: 0
|
||||
SubPlan 3
|
||||
-> Function Scan on pg_options_to_table pg_options_to_table_2 (cost=0.00..0.05 rows=1 width=8) (actual rows=0 loops=247)
|
||||
Filter: (option_name = 'autovacuum_vacuum_scale_factor'::text)
|
||||
Rows Removed by Filter: 0
|
||||
Planning:
|
||||
Buffers: shared hit=38
|
||||
Planning Time: 6.526 ms
|
||||
Execution Time: 15.184 ms
|
||||
(53 rows)
|
||||
|
||||
COMMIT
|
||||
@@ -0,0 +1,885 @@
|
||||
# UKMesh fix review — 2026-10-02
|
||||
|
||||
Branch: `fix/ukmesh-burn-20261002`. Base: `73ee004` (`ukmesh-w5`).
|
||||
Scope: the UKMesh burn brief, including the stretch queue. All edits and local
|
||||
verification stay in this worktree. No deployment, container restart, migration,
|
||||
live-tree access, or environment change is authorized or performed.
|
||||
|
||||
## Baseline and evidence
|
||||
|
||||
- Read `knowledge.md`, `AI_MEMORY.md`, and the full supplied brief, and consulted
|
||||
OpenViking. Tokensave tools and `.tokensave/tokensave.db` are absent here, so
|
||||
structural investigation uses scoped source searches and Git objects.
|
||||
- The initial worktree was clean. `git symbolic-ref refs/remotes/origin/HEAD`
|
||||
fails because the local symbolic ref is absent. `git ls-remote --symref origin
|
||||
HEAD` and `gh repo view gadgethd/ukmesh --json defaultBranchRef,nameWithOwner`
|
||||
both identify `main` (remote HEAD `74d3ff1`). The requested integration base
|
||||
remains `ukmesh-w5`; changing it would change the brief's scope.
|
||||
- Dependencies were missing; `npm ci` in `backend` and `frontend` installed the
|
||||
committed lockfiles without changing them. Initial backend `npm test` after
|
||||
installation: **327/327 pass, 0 fail, 0 skipped**.
|
||||
- The brief describes additional untracked live-checkout code. Neither
|
||||
`runState.test.ts` nor `ownerLastHopPrewarm.ts` exists on the initial branch.
|
||||
Git history contains the former and its full policy in `d834ba8`, and an
|
||||
earlier feed measurement implementation in `333f180`. No live checkout was
|
||||
inspected or copied.
|
||||
- `push.md` is ignored but absent from this worktree and all Git history. An
|
||||
optional location question received no answer during the work; OpenViking
|
||||
supplies no copy. Its exact discipline cannot be verified. The authorized
|
||||
fallback is a clean, reviewed and tested private branch, a non-forced push to
|
||||
that branch only, and an exact remote-SHA check. No merge or deployment follows.
|
||||
- Issue numbers in the brief are not assumed to be GitHub issue IDs:
|
||||
`gh issue view 438 --repo gadgethd/ukmesh` finds no such issue. The supplied
|
||||
brief is the requirements source.
|
||||
|
||||
## 1. Unit suite — #536 / #437 / #301
|
||||
|
||||
Decision: option (c), restore the complete intended retired-workload policy,
|
||||
including the missing export and production callers, from the isolated prior
|
||||
fix `d834ba8`. The initial `runState.ts` exactly matches that commit's parent.
|
||||
This is not a dummy export to make a test load: migration 044 explicitly removed
|
||||
the path-history cache, API and worker; `beginAnalysisRun` must reject a new
|
||||
retired lease, and `getAnalysisWorkloadStates` must describe retirement while
|
||||
preserving historical failures for audit. Supported workloads and unexpected
|
||||
active retired runs continue to expose their actual status and errors.
|
||||
|
||||
Evidence: after restoring only the historical test and `workloadPolicy.ts`,
|
||||
`cd backend && node --import tsx --test src/analysis/runState.test.ts` fails with
|
||||
`SyntaxError: ... does not provide an export named 'normalizeRetiredAnalysisState'`.
|
||||
After restoring the source policy wiring, the same command passes **3/3**, with
|
||||
no database needed. `cd backend && npm test` now passes **330/330**, 0 fail,
|
||||
0 skipped; `cd backend && npm run typecheck` passes. Final suite results appear
|
||||
below.
|
||||
|
||||
## 2. Owner last-hop prewarm — #438
|
||||
|
||||
The named file is absent from the integration base. Recreated a prewarmer around
|
||||
the existing `ownerService.getOwnerLastHopStrength` cache and wired it after
|
||||
owner-auth initialization, with lifecycle shutdown draining. This branch exposes
|
||||
one rolling seven-day result (`ownerRepository.fetchLastHopStrength`), not the
|
||||
live brief's two separate windows; no speculative window API was added.
|
||||
|
||||
Each pass enumerates active, verified owner grants, deduplicates nodes per owner,
|
||||
and refreshes at most two owner/node pairs concurrently. Optional
|
||||
`OWNER_LAST_HOP_PREWARM_CONCURRENCY=1` selects sequential work; values above two
|
||||
remain capped at two. The next pass starts 30 minutes after completion, so cold
|
||||
passes cannot overlap. Start/end logs and every ten completed nodes report
|
||||
owners/nodes completed and total, refreshed and failed. A refresh warns at 20s
|
||||
while still in flight; failures are counted and do not stop the pass.
|
||||
|
||||
Forced warm refreshes replace groups in the inclusive latest-bucket query window
|
||||
and retain earlier history. Foreground reads and
|
||||
warm refreshes share in-flight work. Cache keys include all owned nodes because
|
||||
that exclusion set changes the last-hop query; warming a shared node for one
|
||||
owner must not serve another owner's exclusion scope.
|
||||
|
||||
Validation: `cd backend && node --import tsx --test
|
||||
src/owner/ownerLastHopPrewarm.test.ts src/owner/ownerService.test.ts`:
|
||||
**8/8 pass**, covering the concurrency cap, progress/failure/owner accounting,
|
||||
single-flight, deduplication, an in-flight slow failure, shutdown admission,
|
||||
cache ownership scope and retry. `cd backend && npm run typecheck` passes;
|
||||
`cd backend && npm test`: **338/338 pass**, 0 fail, 0 skipped.
|
||||
Production cold-pass time and DB load are not measured.
|
||||
|
||||
Continuation verification adds three scheduler regressions: a long pass cannot
|
||||
overlap and starts its delay only after completion, enumeration failures recover
|
||||
on the next scheduled pass, and shutdown during enumeration admits no refresh.
|
||||
`cd backend && node --import tsx --test src/owner/ownerLastHopPrewarm.test.ts
|
||||
src/owner/ownerService.test.ts`: **11/11 pass**. No scheduler source change was
|
||||
needed; mocked clocks exercise the real scheduling and shutdown paths.
|
||||
|
||||
## 3. Heartbeat alert semantics — #543 / #507
|
||||
|
||||
Confirmed `currentWorkers` set `worker="link"` from `MAX(itm_computed_at)`.
|
||||
Replaced that source with Redis `meshcore:link:v3:worker_heartbeat`. The current
|
||||
worker's independent thread writes Unix seconds every 10s with a **45s TTL**
|
||||
(the brief's 39s is a live observation, not the source's configured TTL).
|
||||
`last_activity_at` still reports ITM write activity separately.
|
||||
|
||||
The Prometheus gauge collects Redis on each scrape, rather than freezing the
|
||||
age until the five-minute health pass. Reads time out at 2s; missing, invalid,
|
||||
excessively future or unreadable timestamps publish `-1`, replacing any earlier
|
||||
healthy value. The queue alert now handles `< 0` as well as `> 180`, with the
|
||||
existing three-minute holdoff. No threshold increase is needed for a genuine
|
||||
ten-second heartbeat. Empty queues do not trigger this alert.
|
||||
|
||||
Validation: `cd backend && node --import tsx --test
|
||||
src/health/workerHeartbeat.test.ts`: **5/5 pass**, including the real Redis key,
|
||||
timestamp validation, expiry/failure, bounded timeout, repeated real metric
|
||||
scrapes and recovery. `npm run typecheck` passes; `npm test`: **343/343 pass**,
|
||||
0 fail, 0 skipped. `.ukmesh-tools/promtool check rules
|
||||
logging/rules/meshcore.yml`: **24 rules valid**. `.ukmesh-tools/promtool test rules
|
||||
logging/rules/meshcore.test.yml`: **7 scenario groups pass**, including batch
|
||||
gaps, missing heartbeats, holdoff, recovery, idle queues and existing stale cases.
|
||||
|
||||
Standalone promtool 3.15.0 was downloaded from the [official release listing](https://prometheus.io/download/?trk=direct)
|
||||
into an ignored worktree-local tools directory, without starting a container.
|
||||
Archive SHA256: `2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0`,
|
||||
matching the official listing. Live metric/alert behavior remains deploy-gated.
|
||||
|
||||
Publisher-side verification now exercises the actual
|
||||
`link_queue_v3.start_worker_heartbeat` thread with recording Redis clients:
|
||||
immediate publication, integer Unix seconds, ten-second refresh requests,
|
||||
45-second expiry requests, failed-write reconnect, client-creation recovery,
|
||||
an already-stopped worker and interruption of a real ten-second wait on
|
||||
shutdown. The five tests use controlled waits for retry/cadence checks and
|
||||
a real event for shutdown; no Redis service or production client is accessed.
|
||||
Exact command: `PYTHONPATH=viewshed-worker python3 -m unittest discover -s
|
||||
viewshed-worker/tests -p test_link_worker_heartbeat.py -v`: **5/5 pass**.
|
||||
This verifies the publisher's Redis command contract, not live Redis expiry
|
||||
or production alert behavior. The worker implementation is unchanged.
|
||||
|
||||
## 4. UK feed virtualizer — #303
|
||||
|
||||
Implemented option (a): natural border-box measurements from `ResizeObserver`,
|
||||
identity-keyed height storage, prefix offsets and a binary visible-range lookup.
|
||||
The previous `76px * index` arithmetic placed spacers according to an assumed
|
||||
height even when metadata wrapped. The 76px estimate now applies only to unseen
|
||||
rows; rendered rows use their actual fractional-pixel heights.
|
||||
|
||||
Stable packet/scope identities retain measurements across filtering and prepends.
|
||||
Width changes invalidate offscreen measurements. Layout measurement and scroll
|
||||
anchoring preserve the visible packet through observer enrichment and new
|
||||
arrivals; a feed at the top remains pinned to live traffic. Both the desktop
|
||||
internal scroller and mobile document scrolling are supported. Details remain
|
||||
outside the packet list.
|
||||
|
||||
Validation: `cd frontend && npm test`: **99/99 pass**, including **2/2** new
|
||||
offset/range tests. `cd frontend && PLAYWRIGHT_PORT_BASE=4273 npx playwright test
|
||||
feed-virtualizer.spec.ts --project=public-desktop`: **2/2 pass** in Chromium,
|
||||
at 390px and 1280px, checking variable heights, contiguous rows and spacers,
|
||||
scroll-anchor preservation, enrichment, prepends, resizing, bottom reachability,
|
||||
filter reset, empty-filter recovery, new arrivals pinned at the top and absence of
|
||||
page errors. Test-only isolated ports avoid borrowing
|
||||
other sessions' servers; default ports are unchanged. The in-app Browser runtime
|
||||
could not connect to its trusted Node service, so these checks use the repository's
|
||||
Playwright CLI. `cd frontend && npm run build` passes with the existing large-chunk
|
||||
warning. The complete fixture-based E2E matrix passes **55/55**, with no retries;
|
||||
production traffic is not tested.
|
||||
|
||||
`cd frontend && npm run lint:css` reports **6 pre-existing duplicate selectors**
|
||||
in unchanged `globals.css`, `map-app.css` and `owner-portal.css`. Their diff against
|
||||
`73ee004` is empty; this change edits no CSS files.
|
||||
|
||||
Continuation adds empty-filter/repopulation and top-pinned arrival assertions at
|
||||
both widths. Each arrival has a distinct packet identity. Only the initial row
|
||||
readiness wait increases from 5s to the existing application's 15s bootstrap
|
||||
budget; row geometry and scroll-anchor assertions retain their previous limits.
|
||||
|
||||
Browser verification also uncovered incomplete test-server isolation: mobile and
|
||||
accessibility tests still navigated to hardcoded ports 4173–4175 even when the
|
||||
config selected another port range. They now share the config's origins. An
|
||||
explicit `PLAYWRIGHT_PORT_BASE` refuses to borrow existing servers. The three
|
||||
Vite test servers also receive separate dependency caches per port range and
|
||||
site role through a test-only config; the production Vite config is unchanged.
|
||||
Installed Vite source confirms optimization replaces the shared dependency cache;
|
||||
concurrent replacement is a source-based explanation for the earlier dynamic
|
||||
module fetch failure, rather than a proven production cause.
|
||||
|
||||
The first full run passed 53/55; both failures passed unchanged in isolation.
|
||||
After cache isolation, a run passed 54/55 with the feed's 5s initial readiness
|
||||
wait expiring before geometry assertions. The expanded feed tests and 15s
|
||||
readiness wait then passed the full **55/55** matrix without retries. A subsequent
|
||||
fresh-cache four-case check passed both feed widths and tablet topology but the
|
||||
mobile map-mode test exceeded its 30s total budget. Its unchanged isolated trace
|
||||
passed in 19.3s and shows two 4,600-node map bootstraps plus an accessibility scan.
|
||||
That test now has a scoped 45s total budget; its functional assertions are
|
||||
unchanged. The final fresh-cache check on ports 4303–4305 passes **4/4**, without
|
||||
retries. These results establish local fixture behavior, not production performance
|
||||
or the absence of every intermittent browser failure.
|
||||
|
||||
## Stretch queue
|
||||
|
||||
### #424 — Compose adoption guard
|
||||
|
||||
The replacement script previously trusted Compose's selected container without
|
||||
checking its adoption metadata. A mocked backend with missing labels reached
|
||||
replacement: before the guard, `TMPDIR="$PWD/.ukmesh-tools/tmp" bash
|
||||
scripts/test-replace-container.sh` failed with `adoption_empty_stops_before_mutation:
|
||||
expected Compose adoption rejection (65), got 0`. This reproduces the config gap
|
||||
locally; it is not a claim about current live Docker state.
|
||||
|
||||
Added a read-only `scripts/check-compose-adoption.sh SERVICE` preflight. It requires
|
||||
exactly one current container with matching project, service, physical working
|
||||
directory and a config-file list containing this checkout's `docker-compose.yml`.
|
||||
`replace-container.sh` checks the requested service and backend before signature
|
||||
checks, pulls, receipts, migration runs or Compose replacement. Gaps stop with
|
||||
exit 65 and an operator-reconciliation message. No automatic adoption or network
|
||||
recreation is attempted. Operators can run the preflight separately when reviewing
|
||||
a deployment; this session runs it only through mocked Docker.
|
||||
|
||||
Validation: `bash -n scripts/check-compose-adoption.sh scripts/replace-container.sh
|
||||
scripts/test-replace-container.sh` passes. `TMPDIR="$PWD/.ukmesh-tools/tmp" bash
|
||||
scripts/test-replace-container.sh`: **6/6 drills pass** (existing rollback and
|
||||
compatibility, plus empty labels, missing config-file label, wrong directory and
|
||||
wrong project). Gap cases assert no pull, migration/replacement or release receipt.
|
||||
Live adoption and operator reconciliation remain unverified and deploy-gated.
|
||||
|
||||
Continuation expands this to **16/16 mocked drills**: wrong-service labels,
|
||||
absent/multiple containers, null/malformed labels, overlay-only config rejection,
|
||||
base-plus-overlay acceptance, and separate frontend/backend preflights. Every
|
||||
adoption rejection also asserts that image signature verification has not begun.
|
||||
A 10,000-service fixture reproduced an additional bug: `config --services |
|
||||
grep -Fxq` under `pipefail` rejected an existing backend with exit 65 when the
|
||||
producer hit SIGPIPE. The check now consumes the entire list with `grep -Fx`
|
||||
redirected to `/dev/null`. The same large-list drill passes after the fix.
|
||||
|
||||
### #447 — Archive/delete inactive nodes of every role
|
||||
|
||||
The existing cleanup selected only `(role IS NULL OR role = 2)` and required an
|
||||
old, non-null `last_mqtt_observer_seen_at`, so roles 1/3 and never-bridged nodes
|
||||
could not enter its archive/delete path. Added a separate `cleanupInactiveNodes`
|
||||
pass to the health worker's existing initial/six-hour maintenance schedule.
|
||||
The original observer policy remains separate.
|
||||
|
||||
Inactive selection considers the latest node, MQTT, path, predicted-online and
|
||||
creation timestamps, and retains nodes with recent network or observer sightings.
|
||||
Source inspection found packet ingestion updates network sightings independently
|
||||
of the source node's main timestamp; those rollups therefore also fence deletion.
|
||||
The threshold stays bounded to 30–365 days. Test-network nodes, recent nodes and
|
||||
records with wholly unknown age remain protected. No prior bridge or role filter
|
||||
is required.
|
||||
|
||||
The existing transaction/advisory lock archives complete node and visibility
|
||||
records into `maintenance_removed_records` before any delete, then removes the
|
||||
two visibility sets and nodes together. Any failure rolls everything back.
|
||||
Authentication and packet-history tables are outside the cleanup. No schema or
|
||||
migration change is needed.
|
||||
|
||||
Validation: `cd backend && node --import tsx --test
|
||||
src/maintenance/staleMqttObservers.test.ts`: **6/6 pass**. `cd backend && npm run
|
||||
typecheck` passes. Supplementary isolated PostgreSQL 18.3 / PGlite 0.5.8 tests
|
||||
execute the actual selection, advisory lock, archive and delete SQL against
|
||||
minimal in-memory fixture tables. They verify roles 1/3, never-bridged nodes,
|
||||
retention by each freshness source, test/unknown-age exclusions, archive readback,
|
||||
preserved auth/history fixtures, and real transaction rollback on injected archive
|
||||
and node-delete errors: **3/3 pass**, 0 skipped.
|
||||
|
||||
Reproduce without a server or repository dependency change, from the worktree root:
|
||||
|
||||
```sh
|
||||
npm install --prefix .ukmesh-tools/pglite --no-package-lock --ignore-scripts @electric-sql/pglite@0.5.8
|
||||
cd backend
|
||||
TEST_NODE_CLEANUP_PGLITE_MODULE="file://$PWD/../.ukmesh-tools/pglite/node_modules/@electric-sql/pglite/dist/index.js" node --import tsx --test src/maintenance/staleMqttObservers.integration.test.ts
|
||||
```
|
||||
|
||||
The optional fixture test is excluded by the normal unit command. It does not
|
||||
apply project migrations or exercise TimescaleDB, the complete production schema,
|
||||
concurrent ingestion, production candidate counts or cleanup cost; those remain
|
||||
unverified. The final-stretch fixtures below expand the original single privacy
|
||||
function to the current node privacy/generation trigger chain and packet-path
|
||||
classifiers, extracted from the existing migration definitions.
|
||||
|
||||
Continuation found a real privacy interaction omitted by the initial fixture.
|
||||
The current `sync_private_node_prefixes()` definition in migration 049 treats
|
||||
deleting a private node as a privacy transition, removes its prefixes and makes
|
||||
historical packets public. Isolated PostgreSQL tests execute that exact function
|
||||
against fixture tables (no application migration or server) and reproduced
|
||||
`is_private: true -> false`, `visibility_ok: false -> true` for both cleanup
|
||||
policies. Both selectors now retain names containing the privacy marker **or**
|
||||
identities present in `private_node_prefixes`, including mismatched name/index
|
||||
states. Age does not revoke the user's privacy choice. Public inactive nodes of
|
||||
all roles and bridge states still take the archive/delete path.
|
||||
|
||||
The SQL fixture now has **7/7 pass**, including the actual privacy trigger for
|
||||
both policies, the exact 30-day boundary, idempotent repeat passes, archive
|
||||
readback and transaction rollback. The optional integration command above is
|
||||
unchanged; `cd backend && node --import tsx --test
|
||||
src/maintenance/staleMqttObservers.test.ts` remains **6/6 pass**. Removing a
|
||||
private identity while retaining its privacy decision requires a separate
|
||||
tombstone design; no schema change is introduced in this constrained branch.
|
||||
|
||||
### Other heartbeat proxy signals — alert-semantics follow-up findings
|
||||
|
||||
Scoped audit command: `rg -n 'worker_heartbeat_age_seconds|worker_heartbeat_timestamp_seconds|setHeartbeatAge' backend/src viewshed-worker logging`.
|
||||
`docker-compose.yml:811` mounts `logging/rules` into Prometheus's rules directory;
|
||||
`logging/prometheus.yml:5` loads `/etc/prometheus/rules/meshcore.yml`. The edited rule is
|
||||
therefore the repository's active Compose rule source; deployed mounts were not
|
||||
inspected.
|
||||
|
||||
| Signal | Source and cadence | Alert consumer / finding |
|
||||
| --- | --- | --- |
|
||||
| `worker="link"` age | Previously `MAX(node_links.itm_computed_at)`; now the independent Redis heartbeat, collected per scrape. | `ActiveQueueWorkerHeartbeatStale` fixed in this branch; DB write activity remains separate. |
|
||||
| `worker="health"` age | `currentWorkers` reads the previous `MAX(worker_health_snapshots.ts)` before `captureWorkerHealthSnapshot` writes the current snapshot (`backend/src/health/status.ts:378`, `:519`). `backend/src/workers/health.ts:9` schedules the next capture five minutes after completion. | `HealthWorkerHeartbeatStale` still uses `>180` or `<0` for 3m (`logging/rules/meshcore.yml:126`). A normal previous-snapshot age is about 300s and remains frozen between captures. **Source-based inference: another false-positive candidate.** Follow up with a genuine health-process heartbeat and producer/job scoping, plus healthy-cadence regression tests. No live firing frequency is claimed. |
|
||||
| `worker="path_learning"` age | `MAX(path_model_calibration.updated_at)` from published calibration (`status.ts:377`, `path-learning/rebuild.ts:440`); default rebuild cadence one hour (`workers/path-learning.ts:7`). | No current rule consumes this label. It describes publication freshness, not process liveness; keep those concepts separate before adding alerts. |
|
||||
| `worker="link_backfill"` age | `MAX(node_links.last_observed)` (`status.ts:385`), the newest observation. | No current rule consumes this label. Observation recency cannot prove a backfill process is alive or making progress. |
|
||||
| Python `meshcore_worker_heartbeat_timestamp_seconds` | Set by `viewshed-worker/worker_metrics.py:58`. Link updates are on the main loop (`worker.py:2516`); the Redis thread is independent (`link_queue_v3.py:587`). Viewshed's independent thread also updates its exported heartbeat (`worker.py:2483`). | No current rule consumes the timestamp metric. It is unsuitable as a ten-second link-heartbeat substitute without changing its producer cadence. `WorkerMetricsDown` checks exporter `up`, which proves reachability rather than work-loop progress. |
|
||||
|
||||
Queue oldest-job age and backup/restore receipt age describe backlog/freshness
|
||||
explicitly, rather than being renamed as liveness. The health-worker issue and
|
||||
the two unused proxy labels are filed here for follow-up, as requested; this
|
||||
branch changes only the link heartbeat semantics.
|
||||
|
||||
## Final-stretch verification
|
||||
|
||||
Revalidated the clean, pushed continuation HEAD `68d018a` before further work.
|
||||
Fresh local runs pass **350/350 backend units**, **99/99 frontend units** and
|
||||
**7/7 isolated cleanup SQL fixtures**, with no failures or skips. The replacement
|
||||
drills remain **16/16**, and all **24 Prometheus rules** remain valid.
|
||||
|
||||
Production scrape labels uncovered another link-alert false-positive path:
|
||||
`and on()` paired every nonempty queue with any stale heartbeat across all
|
||||
reporters. Three new Prometheus scenarios failed before the fix: a backend Redis
|
||||
failure paged for a healthy health-worker queue, an idle stale reporter paged for
|
||||
a healthy same-job instance, and one stale reporter paged for both queues.
|
||||
The rule now matches `job` and `instance`, keeping queue evidence and heartbeat
|
||||
evidence from the same exporter. The threshold and holdoff are unchanged.
|
||||
`.ukmesh-tools/promtool test rules logging/rules/meshcore.test.yml` passes all
|
||||
**10 scenario groups**, including scoped true-stale alert holdoff and recovery.
|
||||
`.ukmesh-tools/promtool check rules logging/rules/meshcore.yml` passes **24 rules**.
|
||||
This reproduces rule semantics locally, not live alert frequency.
|
||||
|
||||
Expanded both feed browser cases to switch across the 860px breakpoint in both
|
||||
directions while scrolled, checking that the visible packet identity survives
|
||||
the change between document and internal scrolling. The desktop case also
|
||||
filters a scrolled list down to one row, asserts its scroll reset and unchanged
|
||||
page position, then restores the feed. No virtualizer source change was needed.
|
||||
`cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test
|
||||
feed-virtualizer.spec.ts --project=public-desktop --workers=2
|
||||
--trace=retain-on-failure --output=../.ukmesh-tools/stretch-feed-results`:
|
||||
**2/2 pass**, no retries or page errors.
|
||||
|
||||
Owner-cache verification now checks a real warm refresh: inclusive latest-bucket
|
||||
updates replace the old aggregate, earlier history remains, and the rolling
|
||||
seven-day cutoff removes expired points. A failed warm refresh keeps useful
|
||||
foreground data and permits retry; TTL expiry starts a fresh bounded-window
|
||||
query rather than reusing an old cursor. These use fixed mocked dates and require
|
||||
no source change. `cd backend && node --import tsx --test
|
||||
src/owner/ownerLastHopPrewarm.test.ts src/owner/ownerService.test.ts`: **14/14 pass**.
|
||||
|
||||
The expanded cleanup fixture found a second privacy interaction: the
|
||||
`private_node_prefixes` foreign key's `ON DELETE CASCADE` fires its statement
|
||||
trigger even when a deleted public node has no prefixes. A separate minimal
|
||||
PostgreSQL experiment records **trigger depth 1**. Migration 042's depth guard
|
||||
therefore treats this as direct prefix maintenance, advances the public
|
||||
generation and leaves the packet materialization fence stale. The isolated
|
||||
current-trigger fixture reproduced **generation 2 -> 3, fence remaining 2**.
|
||||
PostgreSQL documents that [foreign-key referential actions execute ordinary SQL
|
||||
and fire referencing-table triggers](https://www.postgresql.org/docs/18/trigger-definition.html).
|
||||
|
||||
Both cleanup policies now lock the public visibility state before archival and
|
||||
deletion. Since selected identities have no privacy marker/prefix and
|
||||
their node rows remain locked, this removal cannot change packet privacy bits.
|
||||
Only an **already-current** materialization fence is carried forward to the
|
||||
new public generation, in the same transaction. An existing mismatch or missing
|
||||
materialization is never certified. Visibility-lock failure admits no archive
|
||||
or delete; fence-update failure rolls back both. No migration or trigger
|
||||
definition is changed.
|
||||
|
||||
The fixture installs the exact relevant functions from migrations 042/049/051,
|
||||
with production node/prefix trigger names, prefix FK, and packet-path classifier.
|
||||
It checks both cleanup policies, retained private packets and packet paths,
|
||||
preserved mismatched-prefix states, current fences after public deletion, and
|
||||
real rollback on archive/delete/fence-update errors. Archive replay into the
|
||||
disposable fixture restores complete node and both visibility records, including
|
||||
nullable fields and a populated additional hardware column. The optional SQL
|
||||
command above now passes **10/10**, with no skips; the focused cleanup unit
|
||||
command passes **9/9**. Full backend `npm test`: **356/356**, no failures or skips.
|
||||
Backend typecheck/build, frontend build and the **63 API + 11 operator** contract
|
||||
check pass. Concurrent production ingestion and visibility-lock duration remain
|
||||
unmeasured; this is stronger isolated verification, not a live database claim.
|
||||
|
||||
A controlled-arrival SQL fixture then reproduced another inactivity gap:
|
||||
fresh source-node RF/observer sightings inserted between the initial candidate
|
||||
snapshot and the visibility lock were still deleted (**4 nodes removed instead
|
||||
of 2**). `packetBatch.ts` takes `FOR KEY SHARE` on the same visibility row before
|
||||
updating those rollups. Inactive cleanup now takes its `FOR UPDATE` lock and
|
||||
rechecks the original candidates against the complete freshness/privacy predicate
|
||||
before archival; the row lock fences later packet-ingestion batches until commit.
|
||||
Candidates withdrawn by new sightings produce neither removal archives nor
|
||||
deletes. An all-withdrawn pass returns the usual empty result. The old MQTT
|
||||
observer policy retains its own predicate.
|
||||
|
||||
The controlled arrival passes against actual SQL after the fix, and the full
|
||||
optional fixture suite passes **11/11**, with no skips. This models the query
|
||||
boundary in one disposable connection; it is not a concurrent-ingestion load
|
||||
test. `cd backend && node --import tsx --test
|
||||
src/maintenance/staleMqttObservers.test.ts`: **10/10**. Backend typecheck passes;
|
||||
`cd backend && npm test`: **357/357**, no failures or skips.
|
||||
|
||||
The owner cache responsiveness regression failed before the fix: a fresh
|
||||
foreground cache hit joined a pending forced prewarm query and did not resolve
|
||||
until that query finished. Fresh foreground reads now return the existing cache
|
||||
immediately. Cold/expired reads and forced refreshes still share in-flight work.
|
||||
The test holds a background refresh open, checks the immediate cached response,
|
||||
checks overlapping warm requests issue one query, and then checks the refreshed
|
||||
result. Focused owner/prewarm tests pass **15/15**; backend typecheck and full
|
||||
`npm test` pass **358/358**, with no failures or skips.
|
||||
|
||||
At this safety checkpoint all implementation changes were committed and pushed
|
||||
on the private branch. The expanded browser run on isolated ports 4363–4365
|
||||
finished **54/55**: the multi-viewport mobile map test timed out at `page.goto`
|
||||
with `waitUntil: 'networkidle'`. Its retained trace and test-server output
|
||||
were investigated and resolved in the following sections; the later complete
|
||||
61-case run passed. The two feed cases passed at this checkpoint, including
|
||||
the new breakpoint and filter checks.
|
||||
|
||||
The viewport trace showed the first two of seven sequential map navigations
|
||||
consuming about 15 seconds of one 30-second test budget, including public
|
||||
OpenFreeMap requests. The viewport regression is now seven independently timed
|
||||
cases; each still loads the map from scratch, checks all geometry, and verifies
|
||||
UK-only API requests. No timeout or geometry tolerance was increased.
|
||||
`cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test
|
||||
test/e2e/mobile.spec.ts --grep 'live map stays inside' --workers=2
|
||||
--trace=retain-on-failure --output=../.ukmesh-tools/stretch-viewport-results`:
|
||||
**7/7 pass**, no retries. The complete matrix now contains 61 cases and will be
|
||||
rerun.
|
||||
|
||||
Strengthening the terrain check reproduced a test-harness gap: local DEM
|
||||
requests returned **200 text/html**, and the old RF PNG also produced image
|
||||
decode errors. Shared test routes now serve a valid 256px RGB flat Terrarium
|
||||
tile (zero metres) and a valid 256px RGBA RF tile. OpenFreeMap style/metadata
|
||||
responses are local; the metadata race still delays the response by 1.5s.
|
||||
The two RF metadata stages now stop advancing their revision after completion.
|
||||
Terrain checks require successful PNG responses, decode both images in the
|
||||
browser, check dimensions and reject terrain/RF tile-load error messages.
|
||||
Real DEM rendering takes more time than rejecting HTML; that test has a 60s
|
||||
budget, with existing UI assertion timeouts unchanged.
|
||||
|
||||
`cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test
|
||||
test/e2e/dashboard.spec.ts --grep 'RF coverage remains available with 3D terrain|metadata winning'
|
||||
--workers=1 --trace=retain-on-failure
|
||||
--output=../.ukmesh-tools/stretch-terrain-serial-results`: **4/4 pass**, no retries.
|
||||
A prior parallel run of the stricter checks had startup timeouts and a dynamic
|
||||
import failure with `ERR_NETWORK_CHANGED`; its traces retain that evidence.
|
||||
The final matrix uses one browser worker to reduce local software-rendering
|
||||
contention. These synthetic tiles verify loading/decoding behavior, not real
|
||||
terrain elevation accuracy or production rendering performance.
|
||||
|
||||
A metrics-scrape regression also reproduced a synchronous Redis client-factory
|
||||
failure escaping before the bounded heartbeat read: the whole registry scrape
|
||||
threw instead of replacing a previous healthy link value. The collector now
|
||||
catches initialization failure and exports **-1**, while keeping gauge writes
|
||||
outside that catch. The next scrape can initialize successfully and recover.
|
||||
`cd backend && node --import tsx --test src/health/workerHeartbeat.test.ts`:
|
||||
**6/6 pass**; backend typecheck passes. The first full backend rerun was
|
||||
**358/359** because the unchanged worker-pool replacement test timed out its
|
||||
fresh worker startup inside a 750ms budget under concurrent verification load.
|
||||
The worker-pool correction and passing full rerun are described below.
|
||||
|
||||
The worker-pool regression now controls the 750ms deadline with Node's mock
|
||||
clock rather than imposing that same wall-time budget on replacement thread
|
||||
startup. It checks that active and queued jobs remain pending at 749ms, both
|
||||
expire at 750ms, queues drain, and a real replacement worker returns a new job.
|
||||
Its independent test-runner deadline remains 10 seconds.
|
||||
`cd backend && node --import tsx --test src/path-beta/workerPool.test.ts`:
|
||||
**3/3 pass**. The full backend rerun then passed **359/359**, no failures or
|
||||
skips. No production worker-pool timeout or scheduling code changed.
|
||||
|
||||
The first complete matrix with real raster fixtures finished **59/61**. The
|
||||
remaining traces showed the owner first-load assertion checking while the page
|
||||
still displayed `Loading...`, and the desktop map's post-reload assertion
|
||||
looking for Diagnose before the second bootstrap had finished. Both checks had
|
||||
only the default 5-second assertion budget. Owner tests now use the shared map
|
||||
metadata fixture and a valid runtime-config response; first-load readiness gets
|
||||
15 seconds. The map-mode test explicitly awaits Live Map after reload using the
|
||||
same 15-second readiness check as its initial load, before testing persistence.
|
||||
Polling/lifecycle and mode-state assertions keep their normal budgets.
|
||||
`cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test
|
||||
test/e2e/owner.spec.ts test/e2e/dashboard.spec.ts
|
||||
--grep 'session polling|owner map construction|map modes' --workers=1
|
||||
--trace=retain-on-failure --output=../.ukmesh-tools/stretch-bootstrap-results`:
|
||||
**4/4 pass**, no retries. Backend build also passes after the collector fix.
|
||||
The subsequent complete matrix passed 61/61, as recorded below.
|
||||
|
||||
Feed geometry now also checks **1,413 fractional viewport/overscan windows**
|
||||
across 50 mixed-height rows, including positions before/after the list and
|
||||
overscan larger than the entire list. A direct row-intersection oracle checks
|
||||
every visible row is retained; rendered rows plus top/bottom spacers must equal
|
||||
the complete measured height. `cd frontend && node --import tsx --test
|
||||
src/hooks/measuredVirtualRows.test.ts`: **3/3 pass**. Full frontend `npm test`:
|
||||
**100/100**, no failures or skips. No feed runtime code changed in this step.
|
||||
|
||||
The complete browser rerun with explicit readiness and valid local raster
|
||||
fixtures passed **61/61**, no retries or skips, in 5.6 minutes:
|
||||
`cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test --workers=1
|
||||
--trace=retain-on-failure --output=../.ukmesh-tools/stretch-matrix-ready-results`.
|
||||
Frontend TypeScript/Vite build also passes after the new geometry checks, with
|
||||
the existing chunk-size warning. All code/test changes are committed and pushed;
|
||||
the working tree is clean at this checkpoint. The burn remains active until
|
||||
the requested stop signal.
|
||||
|
||||
Further lock-order verification reproduced **40P01 deadlocks** for both cleanup
|
||||
policies on a disposable native PostgreSQL **16.14** cluster: cleanup locked
|
||||
nodes before waiting for visibility, while a separate ingestion transaction
|
||||
held visibility `FOR KEY SHARE` and then updated the same node. A third case
|
||||
reproduced a busy-node lock timeout (**55P03**). The cluster uses a fresh ignored
|
||||
data directory, an allocated loopback port, synthetic credentials and automatic
|
||||
shutdown/removal. No existing database or container is used. The standalone
|
||||
test tool is [embedded-postgres](https://github.com/leinelissen/embedded-postgres)
|
||||
`16.14.0-beta.17`, installed only under ignored tooling.
|
||||
|
||||
Both policies now discover candidates without node locks, lock visibility,
|
||||
then re-evaluate the complete original predicate and lock eligible nodes with
|
||||
`FOR UPDATE SKIP LOCKED`. This matches ingestion's lock order and avoids waiting
|
||||
on busy row writers that may themselves need visibility. Fresh/private changes
|
||||
withdraw their candidates; busy nodes remain for a later pass. Archival and
|
||||
deletion still happen only after successful revalidation and node locking.
|
||||
|
||||
Native lock tests changed from **0/3** to **3/3 pass**, no skips:
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
TEST_NODE_CLEANUP_POSTGRES_MODULE="file://$PWD/../.ukmesh-tools/embedded-postgres/node_modules/embedded-postgres/dist/index.js" node --import tsx --test src/maintenance/staleMqttObservers.native.integration.test.ts
|
||||
```
|
||||
|
||||
The native fixtures execute the actual cleanup function with simultaneous SQL
|
||||
connections and the visibility/node locking pattern from `packetBatch.ts`.
|
||||
They do not execute the entire ingestion pipeline or measure live workloads.
|
||||
The PGlite boundary hook now identifies the unlocked discovery query; its RF
|
||||
freshness assertions remain intact. Two added boundary cases use the actual
|
||||
privacy triggers to retain new consent and private packet flags for both
|
||||
policies. Those consent cases remain controlled single-connection fixtures.
|
||||
The optional PGlite suite passes **13/13**; focused cleanup units pass **11/11**;
|
||||
full backend `npm test` passes **360/360**, all without failures or skips.
|
||||
Backend typecheck passes. Production lock duration/candidate volume and other
|
||||
concurrent writers remain unmeasured.
|
||||
|
||||
Two additional native cases keep the node timestamps stale while a competing
|
||||
visibility-pinned transaction commits only a source-network or observer-region
|
||||
RF sighting. Inactive cleanup observes each new sighting after acquiring
|
||||
visibility and withdraws the node without an archive or delete. The native
|
||||
suite now passes **5/5**, with no skips; the same command above applies.
|
||||
Backend typecheck passes, and all temporary native database directories are
|
||||
removed after the run.
|
||||
|
||||
Four more native cases commit a new privacy prefix or private-name marker while
|
||||
each cleanup policy waits for an exclusive visibility lock. Both policies
|
||||
observe the committed consent, retain the node and leave the archive empty.
|
||||
The native suite passes **9/9**, no skips; backend typecheck passes and no
|
||||
temporary native database directories remain. These cases model the consent
|
||||
writer's visibility-lock protocol with minimal tables; the separate SQL fixture
|
||||
suite executes the full production privacy trigger chain.
|
||||
|
||||
Two reverse-order cases execute the production
|
||||
`lock_packet_visibility_for_node_privacy_change` BEFORE function. Cleanup first
|
||||
locks visibility; a competing private-name UPDATE locks the node and then waits
|
||||
inside that trigger. Both cleanup policies skip the busy row, commit and release
|
||||
visibility, allowing consent to commit without a deadlock or archive. A later
|
||||
pass retains the now-private node. The native suite passes **11/11**, 0 skipped;
|
||||
backend typecheck passes and every fixture directory is removed. Only this
|
||||
BEFORE function is installed in those cases; the remaining production privacy
|
||||
materialization chain is covered by the separate SQL fixture suite.
|
||||
|
||||
Three further native PostgreSQL **16.14** cases now execute that entire relevant
|
||||
production privacy trigger chain. They confirm that the empty public-node
|
||||
prefix FK cascade advances visibility generation, that an already-current
|
||||
packet fence advances with it, and that pre-existing gapped or missing fences
|
||||
remain uncertified. Stored public packet/path flags remain unchanged, and the
|
||||
node is archived before deletion. The native suite passes **14/14**, no skips;
|
||||
the PostgreSQL-WASM suite remains **13/13**, no skips. Both engines now use
|
||||
`cleanupPrivacyFixture.ts`, moved without changing its function/trigger
|
||||
implementation. Backend typecheck passes and no native fixture directories
|
||||
remain. These are isolated schemas/functions, without production migrations,
|
||||
TimescaleDB or live data.
|
||||
|
||||
## Last-hop regrouping during warm refreshes
|
||||
|
||||
Two new regressions reproduced stale groups after successful incremental
|
||||
refreshes. The SQL groups by bucket, peer identity, current name and resolution;
|
||||
the cache previously upserted those keys without removing groups that the new
|
||||
query no longer returned. A peer rename or unresolved-to-resolved transition
|
||||
doubled the affected bucket's sample count, and an empty query retained obsolete
|
||||
groups. The tests failed **2/2** before the fix.
|
||||
|
||||
The cache now replaces all groups at or after the inclusive query cursor while
|
||||
retaining earlier buckets and the rolling seven-day trim. Query scope, cache
|
||||
keys, single-flight and failure preservation are unchanged. This also preserves
|
||||
the repository's current ordering within each refreshed bucket. Exact focused
|
||||
command: `cd backend && node --import tsx --test
|
||||
src/owner/ownerService.test.ts src/owner/ownerLastHopPrewarm.test.ts`: **17/17
|
||||
pass**, 0 failures or skips. Backend typecheck passes. This verifies response
|
||||
replacement with controlled repository results; no live owner query is claimed.
|
||||
|
||||
An optional isolated SQL fixture now executes the actual
|
||||
`createOwnerRepository.fetchLastHopStrength` query through the service. It seeds
|
||||
three current-bucket packets, renames a peer, adds a newly resolvable peer and
|
||||
checks that the refreshed result still has exactly three current samples with
|
||||
the correct averages. Removing those current packets leaves only the earlier
|
||||
cached history. **1/1 passes**, 0 skipped, with:
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
TEST_OWNER_LAST_HOP_PGLITE_MODULE="file://$PWD/../.ukmesh-tools/pglite/node_modules/@electric-sql/pglite/dist/index.js" node --import tsx --test src/owner/ownerService.integration.test.ts
|
||||
```
|
||||
|
||||
The fixture uses PostgreSQL-WASM with minimal identity tables, an identity
|
||||
canonicalizer and a one-hour `time_bucket` shim. It does not exercise the real
|
||||
identity views, alias reconciliation, TimescaleDB extension or production load.
|
||||
Backend typecheck passes with the fixture included.
|
||||
|
||||
The cleanup freshness audit traced `last_path_evidence_at` to trustworthy
|
||||
multibyte flood-path evidence carrying the packet's observation time. The
|
||||
legacy `last_predicted_online_at` writer records resolution time; read-only
|
||||
path evaluation explicitly disables that update. Cleanup conservatively
|
||||
retains both signals, and the SQL fixtures verify that fresh values prevent
|
||||
deletion. Production resolution frequency and resulting retention are unmeasured.
|
||||
|
||||
## Feed verification in Firefox and WebKit
|
||||
|
||||
`frontend/playwright.feed-cross-browser.config.ts` adds an optional profile for
|
||||
the two existing responsive feed cases in Firefox **153.0** and Playwright
|
||||
WebKit **26.5** on Linux. The default profile still lists **61** Chromium cases;
|
||||
the optional profile lists **4**, runs one worker and has zero retries.
|
||||
|
||||
The first cross-browser run passed **1/4**: immediate anchor reads could run
|
||||
before React committed rows after a programmatic scroll. Anchor capture now
|
||||
waits on the normal assertion deadline for a row intersecting the viewport,
|
||||
and preservation checks use a nullable read within their existing poll. The
|
||||
one-pixel position/contiguity limits and virtualization bounds remain intact.
|
||||
The next run passed **3/4**; WebKit kept the search caret visible during the
|
||||
desktop-to-mobile resize. Completing the search interaction with blur before
|
||||
resuming browsing makes that case pass using the existing virtualizer hook.
|
||||
The complete optional profile now passes **4/4**, no skips or retries. The
|
||||
affected Chromium feed cases also pass **2/2**.
|
||||
|
||||
Exact optional-profile command used on this host:
|
||||
|
||||
```sh
|
||||
cd frontend
|
||||
LD_LIBRARY_PATH="$PWD/../.ukmesh-tools/webkit-system-libs/root/usr/lib/x86_64-linux-gnu" PLAYWRIGHT_BROWSERS_PATH="$PWD/../.ukmesh-tools/playwright-browsers" PLAYWRIGHT_PORT_BASE=4413 npx playwright test --config=playwright.feed-cross-browser.config.ts --trace=retain-on-failure --output=../.ukmesh-tools/feed-cross-browser-final-results
|
||||
```
|
||||
|
||||
The browsers were downloaded with `PLAYWRIGHT_BROWSERS_PATH` pointing into
|
||||
ignored tooling. This host required `libevent-2.1-7t64`,
|
||||
`libgstreamer-plugins-bad1.0-0`, `libavif16`, `libgav1-1` and `libyuv0`; `apt
|
||||
download` and `dpkg-deb --extract` placed them under ignored tooling, and their
|
||||
libraries were linked into the ignored WebKit bundle's `sys/lib`. Its launcher
|
||||
replaces `LD_LIBRARY_PATH`, so the bundle links are necessary here. No system
|
||||
package install or environment-file change was made. These Linux engine tests
|
||||
do not establish behavior on a physical Safari/iOS device.
|
||||
|
||||
Additional exact commands:
|
||||
|
||||
```sh
|
||||
cd frontend
|
||||
PLAYWRIGHT_PORT_BASE=4433 npx playwright test feed-virtualizer.spec.ts --project=public-desktop --workers=1 --trace=retain-on-failure --output=../.ukmesh-tools/feed-chromium-final-results
|
||||
npx tsc --ignoreConfig --noEmit --module NodeNext --moduleResolution NodeNext --target ES2022 --skipLibCheck --types node --strict playwright.feed-cross-browser.config.ts test/e2e/feed-virtualizer.spec.ts
|
||||
npm run build
|
||||
```
|
||||
|
||||
Chromium: **2/2 pass**; direct config/test typecheck passes; production TypeScript
|
||||
and Vite build pass with the existing chunk-size warning. TypeScript requires
|
||||
`--ignoreConfig` for this ad hoc file-based check (the first invocation without
|
||||
it reported TS5112). The frontend unit suite remains **100/100**.
|
||||
|
||||
## Final verification and publication
|
||||
|
||||
Source commits: `9a5e838` (retirement), `2d2b425` (prewarm), `236e948`
|
||||
(heartbeat), `1f2bc76` (feed), `4b30eb0` (Compose guard), `6dda582` (cleanup).
|
||||
Continuation commits: `ab5d5b8` (cleanup privacy), `56bfcb6` (Compose pipefail),
|
||||
`2f8f03e` (scheduler regressions), `bbb14a1` (E2E isolation), `fb981e0` (feed
|
||||
filter/arrival regressions), `62d9a9f` (map-mode cold-start budget). Each change
|
||||
stays on the private branch.
|
||||
Further verification commits: `4a73ca0` (reporter-matched alerts), `2d6b703`
|
||||
(feed breakpoint/filter checks), `29c5111` (incremental owner-cache checks),
|
||||
`a030ea7` (cleanup materialization fence), `9c7cb49` (late-sighting recheck),
|
||||
`e754f46` (responsive fresh-cache reads), `2960f5f` (separate viewport cases),
|
||||
`f5ac2c9` (valid raster fixtures), `a705b89` (Redis initialization recovery),
|
||||
`b561632` (deterministic worker deadlines), `6cf1f70` (bootstrap readiness),
|
||||
`3689c01` (fractional feed-range invariants), `77b4ae7` (cleanup lock order),
|
||||
`62597ee` (concurrent RF arrivals), `c09fcc4` (last-hop group replacement),
|
||||
`d67ac63` (actual last-hop SQL), `17a41aa` (committed consent).
|
||||
The final documentation commit records this audit and the validation below.
|
||||
Full unit suites are rerun after it, before publication.
|
||||
|
||||
Checkpoint `ace67e4` passed the complete local Chromium matrix **61/61** in
|
||||
4.7 minutes, with no retries or skips, after the new anchor-readiness/focus
|
||||
fixture changes. Post-commit backend/frontend unit suites passed **362/362**
|
||||
and **100/100**, respectively. The optional Firefox/WebKit profile passed
|
||||
**4/4**, and the affected Chromium feed cases passed **2/2**.
|
||||
After the shared privacy-trigger fixture and native fence cases were committed,
|
||||
`fcac2a9` also passed both full unit suites (**362/362 backend**, **100/100
|
||||
frontend**, no failures or skips) and the backend build.
|
||||
|
||||
| Exact command | Result |
|
||||
| --- | --- |
|
||||
| `cd backend && npm test` | **362/362 pass**, 0 fail, 0 skipped. Expands to the brief's `node --import tsx --test $(find src -name '*.test.ts' ! -name '*.integration.test.ts' -print)`. |
|
||||
| `cd frontend && npm test` | **100/100 pass**, 0 fail, 0 skipped. |
|
||||
| `cd backend && npm run typecheck` | Pass. |
|
||||
| `cd backend && npm run build` | Pass. |
|
||||
| `cd frontend && npm run build` | TypeScript + Vite pass; existing chunk-size warning. |
|
||||
| `cd backend && npm run contract:check` | Pass: **63 API + 11 operator routes** current. |
|
||||
| `cd frontend && PLAYWRIGHT_PORT_BASE=4273 npx playwright test feed-virtualizer.spec.ts --project=public-desktop` | **2/2 pass**. |
|
||||
| `cd frontend && PLAYWRIGHT_PORT_BASE=4273 npx playwright test --workers=2` | Historical **55/55 pass**; the expanded run was **54/55** before the viewport case split and local map fixtures. The later 61-case matrix passes. |
|
||||
| `cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test --workers=1 --trace=retain-on-failure --output=../.ukmesh-tools/stretch-matrix-final-results` | Historical **59/61** before owner/map-reload readiness checks; affected cases and the subsequent complete matrix pass. |
|
||||
| `cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test --workers=1 --trace=retain-on-failure --output=../.ukmesh-tools/stretch-matrix-ready-results` | **61/61 pass**, all four projects, no retries or skips. |
|
||||
| `cd frontend && PLAYWRIGHT_PORT_BASE=4363 npx playwright test --workers=1 --trace=retain-on-failure --output=../.ukmesh-tools/matrix-ace67e4-results` | **61/61 pass** on `ace67e4`, all four projects, no retries or skips, 4.7 minutes. |
|
||||
| Final Chromium feed command above | **2/2 pass** after anchor-readiness/focus fixture changes. |
|
||||
| Optional Firefox/WebKit profile command above | **4/4 pass**, no retries or skips, including both responsive widths in each engine. |
|
||||
| Direct cross-browser config/test typecheck above | Pass with `--ignoreConfig`; first invocation reported TS5112. |
|
||||
| Fresh-cache targeted browser command below | **4/4 pass** with fresh caches; no retries. |
|
||||
| `.ukmesh-tools/promtool check rules logging/rules/meshcore.yml` | **24 rules valid**. |
|
||||
| `.ukmesh-tools/promtool test rules logging/rules/meshcore.test.yml` | **10 scenario groups pass**. |
|
||||
| `PYTHONPATH=viewshed-worker python3 -m unittest discover -s viewshed-worker/tests -p test_link_worker_heartbeat.py -v` | **5/5 publisher lifecycle tests pass**; recording Redis clients, actual heartbeat thread, no live service. |
|
||||
| `bash -n scripts/check-compose-adoption.sh scripts/replace-container.sh scripts/test-replace-container.sh` | Pass. |
|
||||
| `TMPDIR="$PWD/.ukmesh-tools/tmp" bash scripts/test-replace-container.sh` | **16/16 mocked drills pass**. |
|
||||
| Optional isolated cleanup PGlite integration command above | **13/13 pass**, 0 skipped. |
|
||||
| Optional isolated cleanup native PostgreSQL command above | **14/14 pass**, 0 skipped: 11 concurrent-writer cases plus 3 fence cases using the full relevant production trigger chain. |
|
||||
| Optional isolated last-hop PGlite integration command above | **1/1 pass**, 0 skipped; actual repository SQL, peer regrouping, averages and empty refresh. |
|
||||
| `git diff --check 73ee004` | Pass; scoped review found no migration, schema, environment-file, lockfile or unrelated source edits. |
|
||||
| `cd frontend && npm run lint:css` | Existing **6 duplicate-selector failures** in unchanged files; left out of scope. |
|
||||
|
||||
Fresh-cache command, from the worktree root:
|
||||
|
||||
```sh
|
||||
cd frontend
|
||||
PLAYWRIGHT_PORT_BASE=4303 npx playwright test feed-virtualizer.spec.ts public.spec.ts dashboard.spec.ts --project=public-desktop --project=dashboard-mobile --grep 'measured feed rows|tablet topology|map modes' --workers=2 --trace=retain-on-failure --output=../.ukmesh-tools/cold-final-results
|
||||
```
|
||||
|
||||
Completion audit against the supplied brief:
|
||||
|
||||
| Requirement | Reviewable result and evidence |
|
||||
| --- | --- |
|
||||
| Retired-workload unit-suite repair | Historical policy and current callers restored; 3 focused tests and the complete backend suite pass. |
|
||||
| Owner prewarm observability and bounded load | Progress every ten nodes, in-flight 20s warning, configurable cap of two, ownership-scoped single-flight, responsive fresh-cache reads, replacement of renamed/reclassified groups and shutdown; 17 focused tests pass. Live cold-pass time/load remains unmeasured. |
|
||||
| Link heartbeat source and alert semantics | Per-scrape genuine Redis heartbeat, including client-initialization failure/recovery; 6 source/metric tests, 5 publisher lifecycle tests and 10 Prometheus scenario groups, including independent scrape reporters. Live alert firing is not claimed. |
|
||||
| Variable-height UK feed | ResizeObserver measurements and anchored offsets at both widths; 3 frontend geometry tests, 2 Chromium feed cases and 4 Firefox/WebKit cases pass. |
|
||||
| Compose adoption guard | Local failing mocked reproduction, pre-mutation rejection, 16 passing replacement drills, including the pipefail regression. No live Docker inspection. |
|
||||
| Inactive-node archive/delete for every role | Role/bridge-independent selection, archive-before-delete transaction, private identity retention, late-sighting revalidation, visibility-before-node lock order and preservation of a current privacy fence; 11 unit, 13 PostgreSQL-WASM and 14 native PostgreSQL fixtures pass. No live deletion or migration. |
|
||||
| Final verification and private-branch publication | Full backend/frontend units run on the final documentation HEAD; non-forced branch push and matching remote SHA required before handoff. |
|
||||
| Other heartbeat proxies | Source/cadence/alert-consumer findings recorded above; the health-worker false-positive candidate remains a source-based follow-up. |
|
||||
|
||||
Publication command: `git push -u origin fix/ukmesh-burn-20261002`.
|
||||
Verify with `git rev-parse HEAD` and `git ls-remote --heads origin
|
||||
fix/ukmesh-burn-20261002`; the two full SHAs must match. Remote `ukmesh-w5`
|
||||
remains `73ee00406e7cb0b7251220abbff73144615a8b90`. No force push is used.
|
||||
The operator reviews against that integration base; opening a draft PR is optional.
|
||||
Inspected workflow triggers: branch pushes and pull requests request CI. An
|
||||
earlier `gh run list --branch` listing showed no runs; the explicit repository
|
||||
Actions API subsequently returned the branch's actual runs. On `ace67e4`,
|
||||
[Frontend mobile](https://github.com/gadgethd/ukmesh/actions/runs/37048306966)
|
||||
passed; the [pull-request CI](https://github.com/gadgethd/ukmesh/actions/runs/37048306846)
|
||||
backend, frontend and secret-scan jobs passed. The Workers and Compose job and
|
||||
[push CI](https://github.com/gadgethd/ukmesh/actions/runs/37048301559) were still
|
||||
running at the 2026-10-02 18:46 UTC check; a complete CI verdict is not claimed
|
||||
for that checkpoint. These jobs use disposable GitHub runner stacks. All three
|
||||
repository workflows are active. `release.yml` requires a published release or
|
||||
explicit dispatch. No deployment/release workflow is invoked. Current results
|
||||
are linked from the [PR Checks tab](https://github.com/gadgethd/ukmesh/pull/113/checks).
|
||||
Ignored local logs contain TAP/build/browser output, and ignored tooling holds
|
||||
standalone promtool, PGlite and embedded-postgres. None is part of the shipped runtime.
|
||||
Draft PR: https://github.com/gadgethd/ukmesh/pull/113, against `ukmesh-w5`.
|
||||
|
||||
## Stop-signal handoff — 2026-10-02 19:06 UTC
|
||||
|
||||
Ben's wrap-up signal ends the burn. The latest implementation/test checkpoint
|
||||
is `7a1598e660367879a91762f55bb45e8474f53034`, already pushed with a matching
|
||||
remote SHA and a clean worktree. Both full unit suites passed on that commit:
|
||||
**362/362 backend**, **100/100 frontend**, zero failures or skips. The five
|
||||
Python heartbeat-publisher tests pass. The final commit only records this
|
||||
handoff; both full unit suites are rerun on it before the final summary, where
|
||||
its exact SHA and remote verification are reported.
|
||||
|
||||
The four priority items and both implementation stretch items are complete
|
||||
on the private branch. Draft PR **#113** remains open against `ukmesh-w5`.
|
||||
At the 19:06 UTC check, `7a1598e`'s frontend, secret-scan and mobile checks
|
||||
had passed; [backend and Workers/Compose CI](https://github.com/gadgethd/ukmesh/actions/runs/37051638102)
|
||||
were still running. This is not a complete green remote-CI verdict. The final
|
||||
documentation push may supersede these runs; use the PR Checks tab for their
|
||||
current state.
|
||||
|
||||
Remaining work is operator review/merge/deployment and production validation:
|
||||
DB load and lock duration, cleanup candidates, cold prewarm duration, actual
|
||||
Redis freshness and alert recovery, and Compose adoption on the live host.
|
||||
The health-worker snapshot-age/cadence mismatch remains the source-based
|
||||
follow-up documented above. Linux WebKit testing does not verify a physical
|
||||
Safari/iOS device. Existing CSS lint failures and the chunk-size warning remain.
|
||||
The requested `push.md` was unavailable, so its document-specific discipline
|
||||
cannot be verified; pushes were non-forced and remote SHAs were checked.
|
||||
|
||||
Optional deeper coverage was not added after the stop signal: native
|
||||
PostgreSQL archive/delete/fence failure injection for both cleanup policies,
|
||||
and native current/gapped/missing-fence cases for the older MQTT-only policy.
|
||||
The existing PostgreSQL-WASM suite covers the three failure stages and both
|
||||
policies' privacy behavior. Native PostgreSQL coverage remains **14 cases**:
|
||||
11 concurrent-writer cases and three full-trigger fence cases for inactive-node
|
||||
cleanup. No incomplete edits or new production scope remain in the worktree.
|
||||
|
||||
## Deployment gate, limitations and open questions
|
||||
|
||||
The operator must separately review, merge and deploy code/rules. Local tests
|
||||
cannot prove live DB load, cold-pass wall time, Redis freshness in production,
|
||||
alert recovery or container adoption. This session will not perform those
|
||||
operations. No migrations were run, no environment files were changed, and no
|
||||
live checkout or other burn worktree was accessed. `push.md` remains unavailable;
|
||||
the exact requested document-specific push discipline is not claimed.
|
||||
|
||||
---
|
||||
|
||||
# Backlog round 2 — ukmesh (2026-10-03)
|
||||
|
||||
Worktree: `/home/ben/worktrees/meshcore-analytics/backlog2-20261003`
|
||||
Branch: `backlog2-20261003`
|
||||
Base: `b40793f` (`origin/main`, PR #114 re-land)
|
||||
|
||||
This section records the five items assigned in BRIEF-R2-B. The pre-existing
|
||||
review above is preserved; it belongs to an earlier burn and branch.
|
||||
|
||||
## #455 — dead-only vacuum backlog and alerting
|
||||
|
||||
Status: fixed in the analytics repository.
|
||||
|
||||
- `backend/src/health/status.ts` now counts a table when either the existing
|
||||
relative dead-row condition matches or `n_dead_tup > 50000`, so a deleted
|
||||
TimescaleDB chunk is not hidden by a small `n_live_tup` estimate.
|
||||
- `logging/rules/meshcore.yml` adds `DatabaseVacuumBacklog`; its test fixture
|
||||
proves that a table with one live row and 60,000 dead rows fires after 10m.
|
||||
- `docs/operations.md` documents the bounded operator response and explicitly
|
||||
rules out deleting chunks or the database volume.
|
||||
|
||||
Proof: `vacuum health counts dead-only tables and preserves severity thresholds`
|
||||
passes; pinned `promtool check config` and `promtool test rules` both pass.
|
||||
The rule and health gate are repository changes only—deployment and live alert
|
||||
verification remain for the normal operator rollout.
|
||||
|
||||
## #447 — cleanup of role 1/3 and never-bridged nodes
|
||||
|
||||
Status: already fixed on the supplied base; no duplicate patch was added.
|
||||
|
||||
`b40793f` already contains the role/bridge-independent `cleanupInactiveNodes`
|
||||
predicate. `backend/src/maintenance/staleMqttObservers.test.ts` asserts that
|
||||
inactive cleanup has no role or prior-MQTT requirement, and the optional
|
||||
PostgreSQL fixture `PostgreSQL archives inactive roles 1/3 and never-bridged
|
||||
nodes while retaining every fresh clock` covers the reported cases. This
|
||||
verifies the root-cause fix in the branch without claiming that the historical
|
||||
5,287 live rows were deleted during this repo-only task.
|
||||
|
||||
## #426 — corrupted advert-key variants
|
||||
|
||||
Status: fixed for new ingest in the analytics repository.
|
||||
|
||||
`backend/src/mqtt/decodePacket.ts` verifies the complete MeshCore Ed25519 advert
|
||||
signature before identity is trusted. `backend/src/mqtt/client.ts` records the
|
||||
raw packet telemetry but does not expose the advertised identity or call
|
||||
`evaluateAdvert`/`upsertNode` for an invalid signature. Existing bogus rows are
|
||||
not mass-deleted here: that would require a reviewed data-remediation operation
|
||||
outside this no-deploy worktree task.
|
||||
|
||||
Proof: `only a valid Ed25519 advert may establish node identity` passes with a
|
||||
known valid advert and a tampered advert; the advert-count integrity test also
|
||||
guards that upsert remains inside the verification gate. TypeScript and the
|
||||
full backend suite pass.
|
||||
|
||||
## #436 and #435 — Watchtower `/watch`
|
||||
|
||||
Status: omitted from this repository by an explicit source boundary.
|
||||
|
||||
The assigned Watchtower code is in the separate `gadgethd/meshcore-discord-bot`
|
||||
repository (`discord.ts` and the monitor JSON identified by the sweep report),
|
||||
which is not present in this worktree. No analytics-side integration point can
|
||||
change its ambiguous-match wording/candidate suggestions (#436) or refresh
|
||||
monitor display names after node renames (#435). No sibling checkout was read
|
||||
or modified, so these require a separate PR in the Watchtower repository.
|
||||
|
||||
## Validation and constraints
|
||||
|
||||
- Focused backend tests: 20 passing, including #455, #426, and the existing
|
||||
#447 cleanup assertions.
|
||||
- `npm run typecheck`: passed.
|
||||
- `npm test`: 408 passing plus the synthetic alert-receiver check.
|
||||
- Prometheus config and rule tests: passed with the CI-pinned `promtool` image.
|
||||
- No VPS deployment, database cleanup, container restart, merge, or unrelated
|
||||
branch push was performed.
|
||||
|
||||
PR link: https://github.com/gadgethd/ukmesh/pull/117
|
||||
Reference in new issue
Block a user