diff --git a/db.js b/db.js index 9fd04c3d..81dda78d 100644 --- a/db.js +++ b/db.js @@ -67,21 +67,37 @@ db.exec(` created_at TEXT DEFAULT (datetime('now')) ); - CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); `); // --- Determine schema version --- -let schemaVersion = 0; -try { - const row = db.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1').get(); - if (row) schemaVersion = row.version; -} catch {} +let schemaVersion = db.pragma('user_version', { simple: true }) || 0; + +// Migrate from old schema_version table to pragma user_version +if (schemaVersion === 0) { + try { + const row = db.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1').get(); + if (row && row.version >= 3) { + db.pragma(`user_version = ${row.version}`); + schemaVersion = row.version; + db.exec('DROP TABLE IF EXISTS schema_version'); + } + } catch {} +} + +// Detect v3 schema by column presence (handles crash between migration and version write) +if (schemaVersion === 0) { + try { + const cols = db.pragma('table_info(observations)').map(c => c.name); + if (cols.includes('observer_idx') && !cols.includes('observer_id')) { + db.pragma('user_version = 3'); + schemaVersion = 3; + console.log('[migration-v3] Detected already-migrated schema, set user_version = 3'); + } + } catch {} +} // --- v3 migration: lean observations table --- function needsV3Migration() { @@ -98,7 +114,7 @@ function runV3Migration() { console.log('[migration-v3] Starting observations table optimization...'); // a. Backup DB - const backupPath = dbPath + '.pre-v3-backup'; + const backupPath = dbPath + `.pre-v3-backup-${Date.now()}`; try { console.log(`[migration-v3] Backing up DB to ${backupPath}...`); fs.copyFileSync(dbPath, backupPath); @@ -137,15 +153,12 @@ function runV3Migration() { `).run(); console.log(`[migration-v3] Migrated ${result.changes} rows (${Date.now() - stepStart}ms)`); - // d. Drop old table + // d. Drop view, old table, rename stepStart = Date.now(); + db.exec('DROP VIEW IF EXISTS packets_v'); db.exec('DROP TABLE observations'); - console.log(`[migration-v3] Dropped old observations table (${Date.now() - stepStart}ms)`); - - // e. Rename - stepStart = Date.now(); db.exec('ALTER TABLE observations_v3 RENAME TO observations'); - console.log(`[migration-v3] Renamed observations_v3 → observations (${Date.now() - stepStart}ms)`); + console.log(`[migration-v3] Replaced observations table (${Date.now() - stepStart}ms)`); // f. Create indexes stepStart = Date.now(); @@ -158,22 +171,23 @@ function runV3Migration() { console.log(`[migration-v3] Created indexes (${Date.now() - stepStart}ms)`); // g. Set schema version - db.exec('DELETE FROM schema_version'); - db.prepare('INSERT INTO schema_version (version) VALUES (?)').run(3); + + db.pragma('user_version = 3'); schemaVersion = 3; // h. Rebuild view (done below in common code) - // i. VACUUM + // i. VACUUM + checkpoint stepStart = Date.now(); db.exec('VACUUM'); - console.log(`[migration-v3] VACUUM complete (${Date.now() - stepStart}ms)`); + db.pragma('wal_checkpoint(TRUNCATE)'); + console.log(`[migration-v3] VACUUM + checkpoint complete (${Date.now() - stepStart}ms)`); console.log(`[migration-v3] Migration complete! Total time: ${Date.now() - startTime}ms`); return true; } catch (e) { console.error(`[migration-v3] Migration failed: ${e.message}`); - console.error('[migration-v3] Old data should still be intact if observations table was not yet dropped'); + console.error('[migration-v3] Restore from backup if needed: ' + dbPath + '.pre-v3-backup'); // Try to clean up v3 table if it exists try { db.exec('DROP TABLE IF EXISTS observations_v3'); } catch {} return false; @@ -186,7 +200,7 @@ if (!isV3 && needsV3Migration()) { runV3Migration(); } -// If schema_version < 3 and no migration happened (fresh DB or migration skipped), create old-style table +// If user_version < 3 and no migration happened (fresh DB or migration skipped), create old-style table if (schemaVersion < 3) { const obsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").get(); if (!obsExists) { @@ -208,8 +222,8 @@ if (schemaVersion < 3) { CREATE INDEX idx_observations_timestamp ON observations(timestamp); CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); `); - db.exec('DELETE FROM schema_version'); - db.prepare('INSERT INTO schema_version (version) VALUES (?)').run(3); + + db.pragma('user_version = 3'); schemaVersion = 3; } else { // Old-style observations table exists but migration wasn't run (or failed) diff --git a/public/index.html b/public/index.html index 649b711f..4ff65cf6 100644 --- a/public/index.html +++ b/public/index.html @@ -22,9 +22,9 @@ - - - + + + @@ -81,28 +81,28 @@
- - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/perf.js b/public/perf.js index 79606750..852c89f9 100644 --- a/public/perf.js +++ b/public/perf.js @@ -84,6 +84,25 @@ `; } + // SQLite stats + if (server.sqlite && !server.sqlite.error) { + const sq = server.sqlite; + const walColor = sq.walSizeMB > 50 ? 'var(--status-red)' : sq.walSizeMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; + const freelistColor = sq.freelistMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; + html += `

SQLite

+
${sq.dbSizeMB}MB
DB Size
+
${sq.walSizeMB}MB
WAL Size
+
${sq.freelistMB}MB
Freelist
+
${(sq.rows.transmissions || 0).toLocaleString()}
Transmissions
+
${(sq.rows.observations || 0).toLocaleString()}
Observations
+
${sq.rows.nodes || 0}
Nodes
+
${sq.rows.observers || 0}
Observers
`; + if (sq.walPages) { + html += `
${sq.walPages.busy}
WAL Busy Pages
`; + } + html += `
`; + } + // Server endpoints table const eps = Object.entries(server.endpoints); if (eps.length) { diff --git a/server-helpers.js b/server-helpers.js index b068661c..d04c7029 100644 --- a/server-helpers.js +++ b/server-helpers.js @@ -210,7 +210,7 @@ function disambiguateHops(hops, allNodes, maxHopDist) { else if (!prev && next && dNext > MAX_HOP_DIST) { r.unreliable = true; r.lat = null; r.lon = null; } } - return resolved.map(r => ({ hop: r.hop, name: r.name, lat: r.lat, lon: r.lon, pubkey: r.pubkey, ambiguous: !!r.candidates, unreliable: !!r.unreliable })); + return resolved.map(r => ({ hop: r.hop, name: r.name, lat: r.lat, lon: r.lon, pubkey: r.pubkey, known: !!r.known, ambiguous: !!r.candidates, unreliable: !!r.unreliable })); } // Update hash_size maps for a single packet diff --git a/server.js b/server.js index dd2cf027..766f6e9e 100644 --- a/server.js +++ b/server.js @@ -395,6 +395,28 @@ app.get('/api/perf', (req, res) => { slowQueries: perfStats.slowQueries.slice(-20), cache: { size: cache.size, hits: cache.hits, misses: cache.misses, staleHits: cache.staleHits, recomputes: cache.recomputes, hitRate: cache.hits + cache.misses > 0 ? Math.round(cache.hits / (cache.hits + cache.misses) * 1000) / 10 : 0 }, packetStore: pktStore.getStats(), + sqlite: (() => { + try { + const walInfo = db.db.pragma('wal_checkpoint(PASSIVE)'); + const pageSize = db.db.pragma('page_size', { simple: true }); + const pageCount = db.db.pragma('page_count', { simple: true }); + const freelistCount = db.db.pragma('freelist_count', { simple: true }); + const dbSizeMB = Math.round(pageSize * pageCount / 1048576 * 10) / 10; + const freelistMB = Math.round(pageSize * freelistCount / 1048576 * 10) / 10; + const fs = require('fs'); + const dbPath = process.env.DB_PATH || require('path').join(__dirname, 'data', 'meshcore.db'); + let walSizeMB = 0; + try { walSizeMB = Math.round(fs.statSync(dbPath + '-wal').size / 1048576 * 10) / 10; } catch {} + const stats = db.getStats(); + return { + dbSizeMB, + walSizeMB, + freelistMB, + walPages: walInfo[0] ? { total: walInfo[0].busy + walInfo[0].checkpointed, checkpointed: walInfo[0].checkpointed, busy: walInfo[0].busy } : null, + rows: { transmissions: stats.totalTransmissions, observations: stats.totalObservations, nodes: stats.totalNodes, observers: stats.totalObservers }, + }; + } catch (e) { return { error: e.message }; } + })(), }); }); @@ -425,6 +447,19 @@ setInterval(() => { } catch (e) { console.error('[wal] checkpoint error:', e.message); } }, 300000).unref(); +// Daily TRUNCATE checkpoint at 2:00 AM UTC — reclaims WAL file space +setInterval(() => { + const h = new Date().getUTCHours(); + const m = new Date().getUTCMinutes(); + if (h === 2 && m === 0) { + try { + const t0 = Date.now(); + db.db.pragma('wal_checkpoint(TRUNCATE)'); + console.log(`[wal] daily TRUNCATE checkpoint: ${Date.now() - t0}ms`); + } catch (e) { console.error('[wal] TRUNCATE checkpoint error:', e.message); } + } +}, 60000).unref(); + // --- Health / Telemetry Endpoint --- app.get('/api/health', (req, res) => { const mem = process.memoryUsage(); @@ -522,6 +557,9 @@ if (config.mqttSources && Array.isArray(config.mqttSources)) { }); } +if (process.env.NODE_ENV === 'test') { + console.log('[mqtt] Skipping MQTT connections in test mode'); +} else { for (const source of mqttSources) { try { const opts = { reconnectPeriod: 5000 }; @@ -800,6 +838,7 @@ for (const source of mqttSources) { console.error(`MQTT [${source.name || source.broker}] connection failed (non-fatal):`, e.message); } } +} // end NODE_ENV !== 'test' // --- Express --- app.use(express.json()); diff --git a/test-db-migration.js b/test-db-migration.js index 4b22cb7a..19a637a6 100644 --- a/test-db-migration.js +++ b/test-db-migration.js @@ -126,13 +126,13 @@ function runDbModule(dbPath) { process.env.DB_PATH = ${JSON.stringify(dbPath)}; const db = require(${JSON.stringify(path.resolve(__dirname, 'db'))}); const cols = db.db.pragma('table_info(observations)').map(c => c.name); - const sv = db.db.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1').get(); + const sv = db.db.pragma('user_version', { simple: true }); const obsCount = db.db.prepare('SELECT COUNT(*) as c FROM observations').get().c; const viewRows = db.db.prepare('SELECT * FROM packets_v ORDER BY id').all(); const rawObs = db.db.prepare('SELECT * FROM observations ORDER BY id').all(); console.log(JSON.stringify({ columns: cols, - schemaVersion: sv ? sv.version : 0, + schemaVersion: sv || 0, obsCount, viewRows, rawObs @@ -193,7 +193,8 @@ console.log('Migration from old schema:'); assert(vr2.path_json === null, 'null path_json preserved'); // Verify backup file created - assert(fs.existsSync(dbPath + '.pre-v3-backup'), 'backup file exists'); + const backups1 = fs.readdirSync(tmpDir).filter(f => f.includes('.pre-v3-backup-')); + assert(backups1.length === 1, 'backup file exists'); fs.rmSync(tmpDir, { recursive: true }); } @@ -211,7 +212,8 @@ console.log('\nMigration idempotency:'); assert(info.schemaVersion === 3, 'first run migrates to v3'); // Second run — should NOT re-run migration (no backup overwrite, same data) - const backupMtime = fs.statSync(dbPath + '.pre-v3-backup').mtimeMs; + const backups2pre = fs.readdirSync(tmpDir).filter(f => f.includes('.pre-v3-backup-')); + const backupMtime = fs.statSync(path.join(tmpDir, backups2pre[0])).mtimeMs; info = runDbModule(dbPath); assert(info.schemaVersion === 3, 'second run still v3'); assert(info.obsCount === 3, 'rows still intact'); @@ -219,24 +221,24 @@ console.log('\nMigration idempotency:'); fs.rmSync(tmpDir, { recursive: true }); } -// --- Test 3: Backup failure aborts migration --- -console.log('\nBackup failure aborts migration:'); +// --- Test 3: Each migration creates a unique backup --- +console.log('\nUnique backup per migration:'); { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'meshcore-mig-test3-')); const dbPath = path.join(tmpDir, 'test-mig3.db'); createOldSchemaDB(dbPath); - // Create backup path as a directory so copyFileSync fails - fs.mkdirSync(dbPath + '.pre-v3-backup'); - - // Run db.js — migration should abort, old schema preserved const info = runDbModule(dbPath); - // Old schema should be preserved - assert(info.columns.includes('observer_id'), 'old observer_id column preserved'); - assert(info.schemaVersion < 3, 'schema version not updated'); - assert(info.obsCount === 3, 'old rows still intact'); + // Migration should have completed + assert(info.columns.includes('observer_idx'), 'migration completed'); + assert(info.schemaVersion === 3, 'schema version is 3'); + + // A timestamped backup should exist + const backups = fs.readdirSync(tmpDir).filter(f => f.includes('.pre-v3-backup-')); + assert(backups.length === 1, 'exactly one backup created'); + assert(fs.statSync(path.join(tmpDir, backups[0])).size > 0, 'backup is non-empty'); fs.rmSync(tmpDir, { recursive: true }); } diff --git a/test-db.js b/test-db.js index e9fc8af9..fca0a4fa 100644 --- a/test-db.js +++ b/test-db.js @@ -288,9 +288,9 @@ console.log('\nv3 schema:'); assert(typeof viewRow.timestamp === 'string', 'packets_v timestamp is ISO string'); } - // schema_version table exists with version 3 - const sv = db.db.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1').get(); - assert(sv && sv.version === 3, 'schema_version table has version 3'); + // user_version is 3 + const sv = db.db.pragma('user_version', { simple: true }); + assert(sv === 3, 'user_version is 3'); } // --- v3 ingestion: observer resolved via observer_idx --- diff --git a/tools/e2e-test.js b/tools/e2e-test.js index 3162bd48..01b0e8e8 100644 --- a/tools/e2e-test.js +++ b/tools/e2e-test.js @@ -375,7 +375,7 @@ async function main() { if (chList.length > 0) { const someCh = chList[0]; assert(someCh.messageCount > 0, `channel has messages (${someCh.messageCount})`); - const msgResp = (await get(`/api/channels/${someCh.hash}/messages`)).data; + const msgResp = (await get(`/api/channels/${encodeURIComponent(someCh.hash)}/messages`)).data; assert(msgResp.messages.length > 0, 'channel has message list'); assert(msgResp.messages[0].sender !== undefined, 'message has sender'); console.log(` ✓ Channels: ${chList.length} channels\n`);