fix: prevent chart regen on warm restart

backend/src/api/bootstrap/caches.ts

@@ -16,11 +16,12 @@ export const NODE_LINKS_CACHE_TTL_MS = 60_000;
 export const NODE_LINKS_CACHE_STALE_TTL_MS = 5 * 60_000;
 export const PATH_HISTORY_CACHE_TTL_MS = 60_000;
 export const HEALTH_CACHE_TTL_MS = 60_000;
+// Check durable chart freshness every 30 minutes. Regeneration is gated by the
+// persisted snapshot max age below, so a process restart cannot turn an empty
+// in-memory cache into an unnecessary analytical rebuild.
 export const CHARTS_CACHE_TTL_MS = 30 * 60_000;
-// A chart snapshot is refreshed every 30 minutes, but remains usable for six
-// hours if a refresh is slow or fails. Keeping the storage TTL longer than the
-// freshness TTL lets callers receive the last complete, privacy-filtered
-// snapshot while exactly one bounded refresh runs in the background.
+// A complete persisted snapshot remains current for six hours. Once expired it
+// is still the availability fallback while exactly one refresh runs.
 export const CHARTS_CACHE_STALE_TTL_MS = 6 * 60 * 60_000;
 // The owner dashboard polls /owner/live every 10s. A 5s TTL meant every poll (and
 // every node switch) re-ran 9 DB queries cold. Keep it just above the poll interval

backend/src/api/routes.ts

@@ -29,6 +29,7 @@ import {
   getChannelMessageHistory,
   getRecentPacketEvents,
   getRecentPackets,
+  analyticsQuery,
   pool,
   query,
 } from '../db/index.js';
@@ -228,6 +229,7 @@ registerStatsRoutes(router, {
   statsChartsLimiter: STATS_CHARTS_LIMITER,
   networkFilters,
   query,
+  analyticsQuery,
   getPublicVisibilityGeneration,
   maskDecodedPathNodes,
 });

backend/src/api/routes/stats.ts

@@ -39,6 +39,7 @@ type StatsRouteDeps = {
   statsChartsLimiter: ReturnType<typeof import('express-rate-limit').rateLimit>;
   networkFilters: (network?: string, observer?: string, opts?: { includePrivacy?: boolean }) => NetworkFilters;
   query: QueryFn;
+  analyticsQuery: QueryFn;
   getPublicVisibilityGeneration: () => Promise<number>;
   maskDecodedPathNodes: MaskDecodedPathNodesFn;
 };
@@ -48,6 +49,10 @@ export function registerStatsRoutes(router: Router, deps: StatsRouteDeps): void
     networkFilters: deps.networkFilters,
     query: deps.query,
   });
+  const chartRepository = createStatsRepository({
+    networkFilters: deps.networkFilters,
+    query: deps.analyticsQuery,
+  });
 
   const service = createStatsService({
     statsCache: deps.statsCache,
@@ -56,7 +61,11 @@ export function registerStatsRoutes(router: Router, deps: StatsRouteDeps): void
     chartsCacheTtlMs: deps.chartsCacheTtlMs,
     chartsSnapshotStaleTtlMs: deps.chartsSnapshotStaleTtlMs,
     chartsInflight: deps.chartsInflight,
-    repository,
+    repository: {
+      ...repository,
+      fetchChartsData: chartRepository.fetchChartsData,
+      fetchChannelTraffic: chartRepository.fetchChannelTraffic,
+    },
     getPublicVisibilityGeneration: deps.getPublicVisibilityGeneration,
     maskDecodedPathNodes: deps.maskDecodedPathNodes,
   });

backend/src/stats/chartSnapshot.ts

@@ -36,6 +36,7 @@ export function validateChartSnapshotPayload(
   maxAgeMs: number,
   nowMs = Date.now(),
   expectedVisibilityGeneration?: number,
+  options: { allowExpired?: boolean } = {},
 ): ValidChartSnapshot | null {
   if (
     !payload
@@ -65,7 +66,7 @@ export function validateChartSnapshotPayload(
   if (
     !Number.isFinite(generatedAtMs)
     || generatedAtMs > nowMs + CHART_SNAPSHOT_MAX_FUTURE_SKEW_MS
-    || nowMs - generatedAtMs > maxAgeMs
+    || (!options.allowExpired && nowMs - generatedAtMs > maxAgeMs)
   ) {
     return null;
   }

backend/src/stats/statsService.test.ts

@@ -90,7 +90,9 @@ test('completed canonical charts are reused while observer-scoped charts are nev
 });
 
 test('a valid durable chart snapshot serves a cold process without analytical queries', async () => {
-  const generatedAt = new Date(Date.now() - 60_000).toISOString();
+  // Older than the 30-minute in-memory cadence, but still inside the durable
+  // six-hour max age: this is the exact backend-restart regression guard.
+  const generatedAt = new Date(Date.now() - 45 * 60_000).toISOString();
   const durable = {
     snapshot: {
       status: 'complete',
@@ -211,14 +213,30 @@ test('a visibility change at chart publication fails closed without caching the
   assert.equal(chartsCache.size, 0);
 });
 
-test('expired canonical charts are served while one refresh runs in the background', async () => {
+test('expired persisted canonical charts are served while one refresh runs in the background', async () => {
   let resolveRefresh!: (value: ReturnType<typeof emptyChartsData>) => void;
   const refresh = new Promise<ReturnType<typeof emptyChartsData>>((resolve) => {
     resolveRefresh = resolve;
   });
   let chartCalls = 0;
+  const generatedAt = new Date(Date.now() - 2 * 60_000).toISOString();
+  const stale = {
+    snapshot: {
+      status: 'complete',
+      generatedAt,
+      scope: 'ukmesh',
+      visibilityGeneration: 1,
+    },
+    marker: 'last-complete',
+  };
   const repository = {
-    loadChartSnapshot: async () => null,
+    loadChartSnapshot: async () => ({
+      scope_key: 'ukmesh',
+      schema_version: 2,
+      visibility_generation: '1',
+      generated_at: generatedAt,
+      payload: stale,
+    }),
     saveChartSnapshot: async () => true,
     fetchChartsData: async () => {
       chartCalls += 1;
@@ -226,19 +244,17 @@ test('expired canonical charts are served while one refresh runs in the backgrou
     },
     fetchChannelTraffic: async () => ({ rows: [] }),
   } as unknown as StatsRepository;
-  const stale = { snapshot: { generatedAt: '2026-07-11T12:00:00Z' } };
   const chartsCache = new BoundedTtlMap<string, { ts: number; data: unknown }>({
     maxEntries: 2,
     maxWeight: 1024 * 1024,
     ttlMs: 60_000,
   });
   const chartsInflight = new Map<string, Promise<unknown>>();
-  chartsCache.set('ukmesh:v1', { ts: Date.now() - 1_000, data: stale });
   const service = createStatsService({
     statsCache: new Map(),
     statsCacheTtlMs: 60_000,
     chartsCache,
-    chartsCacheTtlMs: 1,
+    chartsCacheTtlMs: 1_000,
     chartsSnapshotStaleTtlMs: 60_000,
     chartsInflight,
     repository,

backend/src/stats/statsService.ts

@@ -436,13 +436,12 @@ export function createStatsService(deps: StatsServiceDeps) {
     };
   }
 
-  function getUsableCachedCharts(key: string): { ts: number; data: unknown } | undefined {
+  function getCachedCharts(key: string): { ts: number; data: unknown } | undefined {
     const cached = chartsCache.get(key);
     if (!cached) return undefined;
     const ageMs = Date.now() - cached.ts;
     if (
       !Number.isFinite(cached.ts)
-      || ageMs > chartsSnapshotStaleTtlMs
       || ageMs < -CHART_SNAPSHOT_MAX_FUTURE_SKEW_MS
     ) {
       chartsCache.delete(key);
@@ -474,6 +473,7 @@ export function createStatsService(deps: StatsServiceDeps) {
           chartsSnapshotStaleTtlMs,
           Date.now(),
           visibilityGeneration,
+          { allowExpired: true },
         );
         const storedGeneratedAtMs = new Date(row.generated_at).getTime();
         if (
@@ -580,18 +580,18 @@ export function createStatsService(deps: StatsServiceDeps) {
     const scope = `${network ?? 'ukmesh'}`;
     const visibilityGeneration = await getPublicVisibilityGeneration();
     const key = `${scope}:v${visibilityGeneration}`;
-    let cached = getUsableCachedCharts(key);
+    let cached = getCachedCharts(key);
     if (!cached) {
       // Durable complete snapshots are loaded before any analytical query. A
       // cold process can therefore answer immediately and refresh in the
       // background, while observer-scoped data never crosses this boundary.
       cached = await loadPersistedCharts(scope, key, visibilityGeneration);
     }
-    if (cached && Date.now() - cached.ts < chartsCacheTtlMs) {
-      // Region counts and series are part of the canonical 30-minute charts
-      // snapshot. Re-querying the seven-day packet window on every cache hit
-      // allowed concurrent page loads to pile up multi-minute scans. Only the
-      // time-dependent health score needs recalculating between snapshots.
+    if (cached && Date.now() - cached.ts <= chartsSnapshotStaleTtlMs) {
+      // A cold process must treat the persisted snapshot's durable max-age as
+      // authoritative. The shorter in-memory cadence is only how often we
+      // check freshness; it must not force a multi-minute rebuild after every
+      // restart. Only the time-dependent region health score changes here.
       const refreshed = refreshCachedRegionHealth(cached.data);
       chartsCache.set(key, { ts: cached.ts, data: refreshed });
       return refreshed;
@@ -610,8 +610,8 @@ export function createStatsService(deps: StatsServiceDeps) {
       throw error;
     }
     if (cached) {
-      // The canonical snapshot is already privacy-filtered. Serve it while the
-      // single coalesced refresh runs, and retain it if that refresh fails.
+      // Even an expired canonical snapshot is complete and privacy-filtered.
+      // Serve it while one refresh runs, and retain it if that refresh fails.
       void promise.catch(() => { /* retain the last successful value */ });
       return refreshCachedRegionHealth(cached.data);
     }
@@ -634,13 +634,20 @@ export function createStatsService(deps: StatsServiceDeps) {
       }
     };
 
-    // Populate the lightweight summary before starting the much larger chart
-    // snapshot. This keeps /api/stats responsive during a cold restart while
-    // the bounded chart queries continue in the background.
+    const coldRegenDeferMs = Math.max(
+      0,
+      Math.min(15 * 60_000, Number(process.env['CHARTS_COLD_REGEN_DEFER_MS'] ?? 60_000) || 0),
+    );
+
+    // Populate the lightweight summary first, then defer the persisted-snapshot
+    // check. Fresh durable rows never regenerate; missing or genuinely stale
+    // rows can regenerate after WS initial-state has had time to warm.
     initialStatsWarmup = new Promise<void>((resolve) => {
       setTimeout(resolve, 5_000);
     }).then(warmStats);
-    void initialStatsWarmup.finally(warmCharts);
+    void initialStatsWarmup.finally(() => {
+      setTimeout(() => void warmCharts(), coldRegenDeferMs).unref();
+    });
     setInterval(warmStats, statsCacheTtlMs);
     setInterval(warmCharts, chartsCacheTtlMs);
   }