perf(cache): skip expiry scans while cached entries are fresh (#86)

backend/src/cache/boundedTtlMap.test.ts

@@ -60,3 +60,38 @@ test('bounded cache overwrite accounts weight once and reports hit/miss/eviction
   });
   cache.shutdown();
 });
+
+test('sweeps respect replacement deadlines, access order, and backwards clock changes', () => {
+  let now = 100;
+  const cache = new BoundedTtlMap<string, string>({
+    maxEntries: 4, maxWeight: 100, ttlMs: 100, now: () => now,
+  });
+  try {
+    cache.set('replaced', 'old');
+    now = 150;
+    cache.set('replaced', 'new');
+    now = 50;
+    cache.set('earlier', 'value');
+    cache.get('replaced'); // LRU order must not determine expiry order.
+    now = 150;
+    cache.sweep();
+    assert.equal(cache.has('earlier'), false);
+    assert.equal(cache.get('replaced'), 'new');
+    now = 200; // The overwritten value's original expiry is harmless.
+    cache.sweep();
+    assert.equal(cache.size, 1);
+    now = 250;
+    cache.sweep();
+    assert.equal(cache.size, 0);
+    assert.equal(cache.weight(), 0);
+    assert.equal(cache.metrics().expiries, 2);
+    cache.clear();
+    now = 0;
+    cache.set('after-clear', 'value');
+    now = 100;
+    cache.sweep();
+    assert.equal(cache.size, 0);
+  } finally {
+    cache.shutdown();
+  }
+});

backend/src/cache/boundedTtlMap.ts

@@ -27,6 +27,9 @@ export class BoundedTtlMap<K, V> extends Map<K, V> {
   private readonly insertedAt = new Map<K, number>();
   private readonly weights = new Map<K, number>();
   private totalWeight = 0;
+  // A lower bound on the next expiry. Deletions may leave it conservatively
+  // early, but writes need not walk the whole cache while every entry is fresh.
+  private nextExpiryAt = Number.POSITIVE_INFINITY;
   private readonly timer: NodeJS.Timeout;
   private hits = 0;
   private misses = 0;
@@ -90,7 +93,9 @@ export class BoundedTtlMap<K, V> extends Map<K, V> {
     this.sweep();
     this.delete(key);
     super.set(key, value);
-    this.insertedAt.set(key, this.options.now?.() ?? Date.now());
+    const insertedAt = this.options.now?.() ?? Date.now();
+    this.insertedAt.set(key, insertedAt);
+    this.nextExpiryAt = Math.min(this.nextExpiryAt, insertedAt + this.options.ttlMs);
     this.weights.set(key, weight);
     this.totalWeight += weight;
     while (this.size > this.options.maxEntries || this.totalWeight > this.options.maxWeight) {
@@ -122,16 +127,21 @@ export class BoundedTtlMap<K, V> extends Map<K, V> {
     this.insertedAt.clear();
     this.weights.clear();
     this.totalWeight = 0;
+    this.nextExpiryAt = Number.POSITIVE_INFINITY;
     this.recordMetric('cleared');
     this.syncMetricGauges();
   }
 
   sweep(now = this.options.now?.() ?? Date.now()): void {
+    if (now < this.nextExpiryAt) return;
+    this.nextExpiryAt = Number.POSITIVE_INFINITY;
     for (const [key, timestamp] of this.insertedAt) {
       if (now - timestamp >= this.options.ttlMs) {
         this.expiries += 1;
         this.recordMetric('expired');
         this.delete(key);
+      } else {
+        this.nextExpiryAt = Math.min(this.nextExpiryAt, timestamp + this.options.ttlMs);
       }
     }
   }

backend/src/cache/policyRegistry.ts

@@ -177,4 +177,10 @@ export const CACHE_POLICY_REGISTRY: readonly CachePolicyRecord[] = Object.freeze
     scope: 'network + hours + limit', invalidation: 'TTL',
     negativeCaching: 'completed rows only', singleFlight: 'not applicable',
   },
+  {
+    source: 'src/tools/benchmarkBoundedCache.ts#cache',
+    disposition: 'request-local-exclusion',
+    scope: 'one standalone benchmark sample', invalidation: 'shutdown after each sample',
+    negativeCaching: 'none', singleFlight: 'synchronous standalone tool',
+  },
 ]);

backend/src/tools/benchmarkBoundedCache.ts

@@ -0,0 +1,29 @@
+/** Run with tsx. Benchmark in an isolated checkout; no services are contacted. */
+import { BoundedTtlMap } from '../cache/boundedTtlMap.js';
+
+function measure(entries: number) {
+  const cache = new BoundedTtlMap<number, number>({
+    maxEntries: entries, maxWeight: 16 * 1024 * 1024,
+    ttlMs: 3_600_000, weightOf: () => 1, now: () => 0,
+  });
+  try {
+    const start = performance.now();
+    for (let key = 0; key < entries; key += 1) cache.set(key, key);
+    const fillMs = performance.now() - start;
+    const updateStart = performance.now();
+    for (let key = 0; key < 2_000; key += 1) cache.set(key, key + 1);
+    return {
+      entries,
+      fillMs: Number(fillMs.toFixed(2)),
+      update2000Ms: Number((performance.now() - updateStart).toFixed(2)),
+      retainedEntries: cache.size,
+    };
+  } finally {
+    cache.shutdown();
+  }
+}
+
+measure(2_000);
+for (const entries of [4_096, 20_000, 50_000]) {
+  console.log(JSON.stringify(measure(entries)));
+}