mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-09-27 02:07:56 +00:00
dborup
fe68991102
Three follow-up improvements to the View Path map, all requested together: - A compact color/symbol legend (route color, approx marker, first- heard ring) replaces the old prose paragraph explaining the same things -- shorter intro text, faster to scan. - A "show only farthest-traveled route" checkbox appears whenever a packet was heard by more than one station, to declutter busy 20+-branch packets. Only added/removed as a Leaflet layer toggle; the primary branch's own markers/polyline are never touched. - Every touched area is now shaded directly on the map (a real polygon when the area was drawn as one, a rectangle fallback for bbox-only areas) instead of only listed as text -- ties the "touched: X, Y" footer line to actual geography. Non-interactive so shapes never steal a click meant for a branch marker. The shading needed PacketPathResponse.TouchedAreas to carry geometry, not just labels -- changed from []string to []TouchedAreaShape (label + polygon or bbox). annotatePacketPathTouchedAreas now looks up the matched area's full config entry via AreaKeyForPoint instead of just its label via AreaForPoint.
5162 lines
173 KiB
Go
5162 lines
173 KiB
Go
package main
|
||
|
||
import (
|
||
"database/sql"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"math"
|
||
"os"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/meshcore-analyzer/dbschema"
|
||
"github.com/meshcore-analyzer/geofilter"
|
||
_ "modernc.org/sqlite"
|
||
)
|
||
|
||
// routeTypeTransport covers TRANSPORT_FLOOD (0) and TRANSPORT_DIRECT (3) —
|
||
// the only route types that carry transport_code_1 (transport-level scope).
|
||
// Per firmware/docs/packet_format.md § Route Types:
|
||
//
|
||
// 0 = TRANSPORT_FLOOD, 1 = FLOOD, 2 = DIRECT, 3 = TRANSPORT_DIRECT.
|
||
//
|
||
// Routes 1 (FLOOD) and 2 (DIRECT) never carry a scope by protocol — they are
|
||
// inherently unscoped and are counted separately in GetScopeStats (#1838).
|
||
const routeTypeTransportSQL = "route_type IN (0, 3)"
|
||
|
||
// routeTypeNonTransportSQL matches FLOOD (1) and DIRECT (2) — non-transport
|
||
// routes that carry no transport_code_1 and are therefore inherently unscoped
|
||
// per MeshCore protocol (#1838).
|
||
const routeTypeNonTransportSQL = "route_type IN (1, 2)"
|
||
|
||
// DB wraps a read-only connection to the MeshCore SQLite database.
|
||
type DB struct {
|
||
conn *sql.DB
|
||
path string // filesystem path to the database file
|
||
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
|
||
hasResolvedPath bool // observations table has resolved_path column
|
||
hasObsRawHex bool // observations table has raw_hex column (#881)
|
||
hasScopeName bool // transmissions.scope_name column exists (#899)
|
||
hasDefaultScope bool // nodes.default_scope column exists (#899)
|
||
hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903)
|
||
hasLastSeen bool // transmissions.last_seen column exists (#1690)
|
||
|
||
// Channel list cache (60s TTL) — avoids repeated GROUP BY scans (#762)
|
||
channelsCacheMu sync.Mutex
|
||
channelsCacheKey string
|
||
channelsCacheRes []map[string]interface{}
|
||
channelsCacheExp time.Time
|
||
}
|
||
|
||
// OpenDB opens a read-only SQLite connection with WAL mode.
|
||
func OpenDB(path string) (*DB, error) {
|
||
dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=WAL&_busy_timeout=5000", path)
|
||
conn, err := sql.Open("sqlite", dsn)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
conn.SetMaxOpenConns(4)
|
||
conn.SetMaxIdleConns(2)
|
||
if err := conn.Ping(); err != nil {
|
||
conn.Close()
|
||
return nil, fmt.Errorf("ping failed: %w", err)
|
||
}
|
||
d := &DB{conn: conn, path: path}
|
||
d.detectSchema()
|
||
return d, nil
|
||
}
|
||
|
||
func (db *DB) Close() error {
|
||
// Checkpoint WAL before closing to release lock cleanly for new processes
|
||
if _, err := db.conn.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
|
||
log.Printf("[db] WAL checkpoint error: %v", err)
|
||
} else {
|
||
log.Println("[db] WAL checkpoint complete")
|
||
}
|
||
return db.conn.Close()
|
||
}
|
||
|
||
// detectSchema checks if the observations table uses v3 schema (observer_idx).
|
||
func (db *DB) detectSchema() {
|
||
rows, err := db.conn.Query("PRAGMA table_info(observations)")
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var cid int
|
||
var colName string
|
||
var colType sql.NullString
|
||
var notNull, pk int
|
||
var dflt sql.NullString
|
||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil {
|
||
if colName == "observer_idx" {
|
||
db.isV3 = true
|
||
}
|
||
if colName == "resolved_path" {
|
||
db.hasResolvedPath = true
|
||
}
|
||
if colName == "raw_hex" {
|
||
db.hasObsRawHex = true
|
||
}
|
||
}
|
||
}
|
||
|
||
txRows, err := db.conn.Query("PRAGMA table_info(transmissions)")
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer txRows.Close()
|
||
for txRows.Next() {
|
||
var cid int
|
||
var colName string
|
||
var colType sql.NullString
|
||
var notNull, pk int
|
||
var dflt sql.NullString
|
||
if txRows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil {
|
||
if colName == "scope_name" {
|
||
db.hasScopeName = true
|
||
}
|
||
if colName == "last_seen" {
|
||
db.hasLastSeen = true
|
||
}
|
||
}
|
||
}
|
||
|
||
nodeRows, err := db.conn.Query("PRAGMA table_info(nodes)")
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer nodeRows.Close()
|
||
for nodeRows.Next() {
|
||
var cid int
|
||
var colName string
|
||
var colType sql.NullString
|
||
var notNull, pk int
|
||
var dflt sql.NullString
|
||
if nodeRows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil {
|
||
switch colName {
|
||
case "default_scope":
|
||
db.hasDefaultScope = true
|
||
case "multibyte_sup":
|
||
db.hasMultibyteSupCols = true
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// nodeSelectCols returns the SELECT column list for nodes queries.
|
||
// When hasDefaultScope is true, default_scope is appended as the last column.
|
||
func (db *DB) nodeSelectCols() string {
|
||
cols := "public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c, foreign_advert"
|
||
if db.hasDefaultScope {
|
||
cols += ", default_scope"
|
||
}
|
||
return cols
|
||
}
|
||
|
||
// transmissionBaseSQL returns the SELECT columns and JOIN clause for transmission-centric queries.
|
||
func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) {
|
||
if db.isV3 {
|
||
selectCols = `t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.decoded_json,
|
||
COALESCE((SELECT COUNT(*) FROM observations WHERE transmission_id = t.id), 0) AS observation_count,
|
||
obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata,
|
||
o.snr, o.rssi, o.path_json, o.direction`
|
||
observerJoin = `LEFT JOIN observations o ON o.id = (
|
||
SELECT id FROM observations WHERE transmission_id = t.id
|
||
ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1
|
||
)
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx`
|
||
} else {
|
||
selectCols = `t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.decoded_json,
|
||
COALESCE((SELECT COUNT(*) FROM observations WHERE transmission_id = t.id), 0) AS observation_count,
|
||
o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata,
|
||
o.snr, o.rssi, o.path_json, o.direction`
|
||
observerJoin = `LEFT JOIN observations o ON o.id = (
|
||
SELECT id FROM observations WHERE transmission_id = t.id
|
||
ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1
|
||
)
|
||
LEFT JOIN observers obs2 ON obs2.id = o.observer_id`
|
||
}
|
||
if db.hasScopeName {
|
||
selectCols += `, t.scope_name`
|
||
}
|
||
return
|
||
}
|
||
|
||
// scanTransmissionRow scans a row from the transmission-centric query.
|
||
// Returns a map matching the Node.js packet-store transmission shape.
|
||
func (db *DB) scanTransmissionRow(rows *sql.Rows) map[string]interface{} {
|
||
var id, observationCount int
|
||
var rawHex, hash, firstSeen, decodedJSON, observerID, observerName, observerIATA, pathJSON, direction sql.NullString
|
||
var routeType, payloadType sql.NullInt64
|
||
var snr, rssi sql.NullFloat64
|
||
var scopeName sql.NullString
|
||
|
||
scanArgs := []interface{}{&id, &rawHex, &hash, &firstSeen, &routeType, &payloadType, &decodedJSON,
|
||
&observationCount, &observerID, &observerName, &observerIATA, &snr, &rssi, &pathJSON, &direction}
|
||
if db.hasScopeName {
|
||
scanArgs = append(scanArgs, &scopeName)
|
||
}
|
||
if err := rows.Scan(scanArgs...); err != nil {
|
||
return nil
|
||
}
|
||
|
||
m := map[string]interface{}{
|
||
"id": id,
|
||
"raw_hex": nullStr(rawHex),
|
||
"hash": nullStr(hash),
|
||
"first_seen": nullStr(firstSeen),
|
||
"timestamp": nullStr(firstSeen),
|
||
"route_type": nullInt(routeType),
|
||
"payload_type": nullInt(payloadType),
|
||
"decoded_json": nullStr(decodedJSON),
|
||
"observation_count": observationCount,
|
||
"observer_id": nullStr(observerID),
|
||
"observer_name": nullStr(observerName),
|
||
"observer_iata": nullStr(observerIATA),
|
||
"snr": nullFloat(snr),
|
||
"rssi": nullFloat(rssi),
|
||
"path_json": nullStr(pathJSON),
|
||
"direction": nullStr(direction),
|
||
}
|
||
if db.hasScopeName {
|
||
m["scope_name"] = nullStr(scopeName)
|
||
}
|
||
return m
|
||
}
|
||
|
||
// Node represents a row from the nodes table.
|
||
type Node struct {
|
||
PublicKey string `json:"public_key"`
|
||
Name *string `json:"name"`
|
||
Role *string `json:"role"`
|
||
Lat *float64 `json:"lat"`
|
||
Lon *float64 `json:"lon"`
|
||
LastSeen *string `json:"last_seen"`
|
||
FirstSeen *string `json:"first_seen"`
|
||
AdvertCount int `json:"advert_count"`
|
||
BatteryMv *int `json:"battery_mv"`
|
||
TemperatureC *float64 `json:"temperature_c"`
|
||
}
|
||
|
||
// Observer represents a row from the observers table.
|
||
type Observer struct {
|
||
ID string `json:"id"`
|
||
Name *string `json:"name"`
|
||
IATA *string `json:"iata"`
|
||
LastSeen *string `json:"last_seen"`
|
||
FirstSeen *string `json:"first_seen"`
|
||
PacketCount int `json:"packet_count"`
|
||
Model *string `json:"model"`
|
||
Firmware *string `json:"firmware"`
|
||
ClientVersion *string `json:"client_version"`
|
||
Radio *string `json:"radio"`
|
||
BatteryMv *int `json:"battery_mv"`
|
||
UptimeSecs *int64 `json:"uptime_secs"`
|
||
NoiseFloor *float64 `json:"noise_floor"`
|
||
LastPacketAt *string `json:"last_packet_at"`
|
||
// Issue #1478: per-observer naive-clock skew tracking.
|
||
// Written by the ingestor in cmd/ingestor/db.go RecordNaiveSkew whenever
|
||
// resolveRxTime clamps a naive envelope timestamp >15 min off UTC. The
|
||
// server reads these as-is; the handler derives the bool `clock_naive`
|
||
// from clock_last_naive_at being within the last 24h.
|
||
ClockSkewSeconds *int64 `json:"clock_skew_seconds"`
|
||
ClockSkewCount24h int `json:"clock_skew_count_24h"`
|
||
ClockLastNaiveAt *string `json:"clock_last_naive_at"`
|
||
// Issue #1290: firmware 1.16 `repeat: on|off` flag persisted by the
|
||
// ingestor. true = relay-capable, false = listener-only, nil =
|
||
// unknown (legacy observer that never sent the field — drives the
|
||
// tri-state UI badge so legacy rows don't masquerade as confirmed
|
||
// repeaters). The ingestor sets can_relay_seen=1 only when it has
|
||
// an explicit value; the read layer returns nil when seen=0.
|
||
CanRelay *bool `json:"can_relay,omitempty"`
|
||
}
|
||
|
||
// Transmission represents a row from the transmissions table.
|
||
type Transmission struct {
|
||
ID int `json:"id"`
|
||
RawHex *string `json:"raw_hex"`
|
||
Hash string `json:"hash"`
|
||
FirstSeen string `json:"first_seen"`
|
||
RouteType *int `json:"route_type"`
|
||
PayloadType *int `json:"payload_type"`
|
||
PayloadVersion *int `json:"payload_version"`
|
||
DecodedJSON *string `json:"decoded_json"`
|
||
CreatedAt *string `json:"created_at"`
|
||
}
|
||
|
||
// Observation (observation-level data).
|
||
type Observation struct {
|
||
ID int `json:"id"`
|
||
RawHex *string `json:"raw_hex"`
|
||
Timestamp *string `json:"timestamp"`
|
||
ObserverID *string `json:"observer_id"`
|
||
ObserverName *string `json:"observer_name"`
|
||
Direction *string `json:"direction"`
|
||
SNR *float64 `json:"snr"`
|
||
RSSI *float64 `json:"rssi"`
|
||
Score *int `json:"score"`
|
||
Hash *string `json:"hash"`
|
||
RouteType *int `json:"route_type"`
|
||
PayloadType *int `json:"payload_type"`
|
||
PayloadVer *int `json:"payload_version"`
|
||
PathJSON *string `json:"path_json"`
|
||
DecodedJSON *string `json:"decoded_json"`
|
||
CreatedAt *string `json:"created_at"`
|
||
}
|
||
|
||
// Stats holds system statistics.
|
||
type Stats struct {
|
||
TotalPackets int `json:"totalPackets"`
|
||
TotalTransmissions int `json:"totalTransmissions"`
|
||
TotalObservations int `json:"totalObservations"`
|
||
TotalNodes int `json:"totalNodes"`
|
||
TotalNodesAllTime int `json:"totalNodesAllTime"`
|
||
TotalObservers int `json:"totalObservers"`
|
||
PacketsLastHour int `json:"packetsLastHour"`
|
||
PacketsLast24h int `json:"packetsLast24h"`
|
||
}
|
||
|
||
// GetStats returns aggregate counts (matches Node.js db.getStats shape).
|
||
func (db *DB) GetStats() (*Stats, error) {
|
||
s := &Stats{}
|
||
err := db.conn.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&s.TotalTransmissions)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
s.TotalPackets = s.TotalTransmissions
|
||
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM observations").Scan(&s.TotalObservations)
|
||
// Node.js uses 7-day active nodes for totalNodes
|
||
sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339)
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM nodes WHERE last_seen > ?", sevenDaysAgo).Scan(&s.TotalNodes)
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&s.TotalNodesAllTime)
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0").Scan(&s.TotalObservers)
|
||
|
||
oneHourAgo := time.Now().Add(-1 * time.Hour).Unix()
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneHourAgo).Scan(&s.PacketsLastHour)
|
||
|
||
oneDayAgo := time.Now().Add(-24 * time.Hour).Unix()
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneDayAgo).Scan(&s.PacketsLast24h)
|
||
|
||
return s, nil
|
||
}
|
||
|
||
// GetDBSizeStats returns SQLite file sizes and row counts (matching Node.js /api/perf sqlite shape).
|
||
func (db *DB) GetDBSizeStats() map[string]interface{} {
|
||
result := map[string]interface{}{}
|
||
|
||
// DB file size
|
||
var dbSizeMB float64
|
||
if db.path != "" && db.path != ":memory:" {
|
||
if info, err := os.Stat(db.path); err == nil {
|
||
dbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10
|
||
}
|
||
}
|
||
result["dbSizeMB"] = dbSizeMB
|
||
|
||
// WAL file size
|
||
var walSizeMB float64
|
||
if db.path != "" && db.path != ":memory:" {
|
||
if info, err := os.Stat(db.path + "-wal"); err == nil {
|
||
walSizeMB = math.Round(float64(info.Size())/1048576*10) / 10
|
||
}
|
||
}
|
||
result["walSizeMB"] = walSizeMB
|
||
|
||
// Freelist size via PRAGMA (matches Node.js: page_size * freelist_count)
|
||
var pageSize, freelistCount int64
|
||
db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize)
|
||
db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount)
|
||
freelistMB := math.Round(float64(pageSize*freelistCount)/1048576*10) / 10
|
||
result["freelistMB"] = freelistMB
|
||
|
||
// WAL checkpoint info (matches Node.js: PRAGMA wal_checkpoint(PASSIVE))
|
||
var walBusy, walLog, walCheckpointed int
|
||
err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed)
|
||
if err == nil {
|
||
result["walPages"] = map[string]interface{}{
|
||
"total": walLog,
|
||
"checkpointed": walCheckpointed,
|
||
"busy": walBusy,
|
||
}
|
||
} else {
|
||
result["walPages"] = map[string]interface{}{
|
||
"total": 0,
|
||
"checkpointed": 0,
|
||
"busy": 0,
|
||
}
|
||
}
|
||
|
||
// Row counts per table
|
||
rows := map[string]int{}
|
||
for _, table := range []string{"transmissions", "observations", "nodes", "observers"} {
|
||
var count int
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count)
|
||
rows[table] = count
|
||
}
|
||
result["rows"] = rows
|
||
|
||
return result
|
||
}
|
||
|
||
// GetDBSizeStatsTyped returns SQLite file sizes and row counts as a typed struct.
|
||
func (db *DB) GetDBSizeStatsTyped() SqliteStats {
|
||
result := SqliteStats{}
|
||
|
||
if db.path != "" && db.path != ":memory:" {
|
||
if info, err := os.Stat(db.path); err == nil {
|
||
result.DbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10
|
||
}
|
||
}
|
||
|
||
if db.path != "" && db.path != ":memory:" {
|
||
if info, err := os.Stat(db.path + "-wal"); err == nil {
|
||
result.WalSizeMB = math.Round(float64(info.Size())/1048576*10) / 10
|
||
}
|
||
}
|
||
|
||
var pageSize, freelistCount int64
|
||
db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize)
|
||
db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount)
|
||
result.FreelistMB = math.Round(float64(pageSize*freelistCount)/1048576*10) / 10
|
||
|
||
var walBusy, walLog, walCheckpointed int
|
||
err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed)
|
||
if err == nil {
|
||
result.WalPages = &WalPages{
|
||
Total: walLog,
|
||
Checkpointed: walCheckpointed,
|
||
Busy: walBusy,
|
||
}
|
||
} else {
|
||
result.WalPages = &WalPages{}
|
||
}
|
||
|
||
rows := &SqliteRowCounts{}
|
||
for _, table := range []string{"transmissions", "observations", "nodes", "observers"} {
|
||
var count int
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count)
|
||
switch table {
|
||
case "transmissions":
|
||
rows.Transmissions = count
|
||
case "observations":
|
||
rows.Observations = count
|
||
case "nodes":
|
||
rows.Nodes = count
|
||
case "observers":
|
||
rows.Observers = count
|
||
}
|
||
}
|
||
result.Rows = rows
|
||
|
||
return result
|
||
}
|
||
|
||
// GetRoleCounts returns count per role (7-day active, matching Node.js /api/stats).
|
||
func (db *DB) GetRoleCounts() map[string]int {
|
||
sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339)
|
||
counts := map[string]int{}
|
||
for _, role := range []string{"repeater", "room", "companion", "sensor"} {
|
||
var c int
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?", role, sevenDaysAgo).Scan(&c)
|
||
counts[role+"s"] = c
|
||
}
|
||
return counts
|
||
}
|
||
|
||
// GetAllRoleCounts returns count per role (all nodes, no time filter — matching Node.js /api/nodes).
|
||
func (db *DB) GetAllRoleCounts() map[string]int {
|
||
counts := map[string]int{}
|
||
for _, role := range []string{"repeater", "room", "companion", "sensor"} {
|
||
var c int
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM nodes WHERE role = ?", role).Scan(&c)
|
||
counts[role+"s"] = c
|
||
}
|
||
return counts
|
||
}
|
||
|
||
// PacketQuery holds filter params for packet listing.
|
||
type PacketQuery struct {
|
||
Limit int
|
||
Offset int
|
||
Type *int
|
||
Route *int
|
||
Observer string
|
||
Hash string
|
||
Since string
|
||
Until string
|
||
Region string
|
||
Area string // area key; filters by transmitting node's GPS position
|
||
Node string
|
||
Channel string // channel_hash filter (#812). Plain names like "#test"/"public" or "enc_<HEX>" for encrypted
|
||
Order string // ASC or DESC
|
||
ExpandObservations bool // when true, include observation sub-maps in txToMap output
|
||
}
|
||
|
||
// PacketResult wraps paginated packet list.
|
||
type PacketResult struct {
|
||
Packets []map[string]interface{} `json:"packets"`
|
||
Total int `json:"total"`
|
||
Limit int `json:"limit"`
|
||
Offset int `json:"offset"`
|
||
}
|
||
|
||
// QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape).
|
||
func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) {
|
||
if q.Limit <= 0 {
|
||
q.Limit = 50
|
||
}
|
||
if q.Order == "" {
|
||
q.Order = "DESC"
|
||
}
|
||
|
||
where, args := db.buildTransmissionWhere(q)
|
||
w := ""
|
||
if len(where) > 0 {
|
||
w = "WHERE " + strings.Join(where, " AND ")
|
||
}
|
||
|
||
// Count transmissions (not observations)
|
||
var total int
|
||
if len(where) == 0 {
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&total)
|
||
} else {
|
||
countSQL := fmt.Sprintf("SELECT COUNT(*) FROM transmissions t %s", w)
|
||
db.conn.QueryRow(countSQL, args...).Scan(&total)
|
||
}
|
||
|
||
// #1345: order by ingest id, NOT first_seen. PR #1233 made first_seen=rxTime,
|
||
// so buffered-then-uploaded observer packets with hours-old rxTime were
|
||
// sorting to the top/middle and hiding fresh ingest. Ordering by id keeps
|
||
// "latest activity" semantically equal to "what we ingested last" — which
|
||
// is what the packets page is showing. The `since=` filter still uses
|
||
// first_seen / observation timestamp, preserving "received-by-radio since X."
|
||
selectCols, observerJoin := db.transmissionBaseSQL()
|
||
querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?",
|
||
selectCols, observerJoin, w, q.Order)
|
||
|
||
qArgs := make([]interface{}, len(args))
|
||
copy(qArgs, args)
|
||
qArgs = append(qArgs, q.Limit, q.Offset)
|
||
|
||
rows, err := db.conn.Query(querySQL, qArgs...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
packets := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
p := db.scanTransmissionRow(rows)
|
||
if p != nil {
|
||
packets = append(packets, p)
|
||
}
|
||
}
|
||
|
||
return &PacketResult{Packets: packets, Total: total}, nil
|
||
}
|
||
|
||
// QueryGroupedPackets groups by hash (transmissions) — queries transmissions table directly for performance.
|
||
func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) {
|
||
if q.Limit <= 0 {
|
||
q.Limit = 50
|
||
}
|
||
|
||
where, args := db.buildTransmissionWhere(q)
|
||
w := ""
|
||
if len(where) > 0 {
|
||
w = "WHERE " + strings.Join(where, " AND ")
|
||
}
|
||
|
||
// Count total transmissions (fast — queries transmissions directly, not a VIEW)
|
||
var total int
|
||
if len(where) == 0 {
|
||
db.conn.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&total)
|
||
} else {
|
||
db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM transmissions t %s", w), args...).Scan(&total)
|
||
}
|
||
|
||
// Build grouped query using transmissions table with correlated subqueries.
|
||
// #1189 R2: distinct_iatas is a NEW column — comma-separated DISTINCT IATA
|
||
// codes across all observers of the transmission, with empty/NULL IATAs
|
||
// excluded. Frontend needs this on the DEFAULT COLLAPSED VIEW (where
|
||
// p._children is empty), so we compute it server-side.
|
||
groupedScopeCol := ""
|
||
if db.hasScopeName {
|
||
groupedScopeCol = ", t.scope_name"
|
||
}
|
||
var querySQL string
|
||
if db.isV3 {
|
||
querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type,
|
||
COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count,
|
||
COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count,
|
||
COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest,
|
||
obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata,
|
||
o.snr, o.rssi, o.path_json,
|
||
COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+groupedScopeCol+`
|
||
FROM transmissions t
|
||
LEFT JOIN observations o ON o.id = (
|
||
SELECT id FROM observations WHERE transmission_id = t.id
|
||
ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1
|
||
)
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
%s ORDER BY latest DESC LIMIT ? OFFSET ?`, w)
|
||
} else {
|
||
querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type,
|
||
COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count,
|
||
COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count,
|
||
COALESCE((SELECT MAX(oi.timestamp) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest,
|
||
o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata,
|
||
o.snr, o.rssi, o.path_json,
|
||
COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+groupedScopeCol+`
|
||
FROM transmissions t
|
||
LEFT JOIN observations o ON o.id = (
|
||
SELECT id FROM observations WHERE transmission_id = t.id
|
||
ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1
|
||
)
|
||
LEFT JOIN observers obs2 ON obs2.id = o.observer_id
|
||
%s ORDER BY latest DESC LIMIT ? OFFSET ?`, w)
|
||
}
|
||
|
||
qArgs := make([]interface{}, len(args))
|
||
copy(qArgs, args)
|
||
qArgs = append(qArgs, q.Limit, q.Offset)
|
||
|
||
rows, err := db.conn.Query(querySQL, qArgs...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
packets := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
var hash, firstSeen, rawHex, decodedJSON, latest, observerID, observerName, observerIATA, pathJSON, distinctIatasCSV sql.NullString
|
||
var payloadType, routeType sql.NullInt64
|
||
var count, observerCount int
|
||
var snr, rssi sql.NullFloat64
|
||
var scopeName sql.NullString
|
||
|
||
scanArgs := []interface{}{&hash, &firstSeen, &rawHex, &decodedJSON, &payloadType, &routeType,
|
||
&count, &observerCount, &latest,
|
||
&observerID, &observerName, &observerIATA, &snr, &rssi, &pathJSON, &distinctIatasCSV}
|
||
if db.hasScopeName {
|
||
scanArgs = append(scanArgs, &scopeName)
|
||
}
|
||
if err := rows.Scan(scanArgs...); err != nil {
|
||
continue
|
||
}
|
||
|
||
packets = append(packets, map[string]interface{}{
|
||
"hash": nullStr(hash),
|
||
"first_seen": nullStr(firstSeen),
|
||
"count": count,
|
||
"observer_count": observerCount,
|
||
"observation_count": count,
|
||
"latest": nullStr(latest),
|
||
"observer_id": nullStr(observerID),
|
||
"observer_name": nullStr(observerName),
|
||
"observer_iata": nullStr(observerIATA),
|
||
"distinct_iatas": parseDistinctIatasCSV(nullStr(distinctIatasCSV)),
|
||
"path_json": nullStr(pathJSON),
|
||
"payload_type": nullInt(payloadType),
|
||
"route_type": nullInt(routeType),
|
||
"raw_hex": nullStr(rawHex),
|
||
"decoded_json": nullStr(decodedJSON),
|
||
"snr": nullFloat(snr),
|
||
"rssi": nullFloat(rssi),
|
||
"scope_name": nullStr(scopeName),
|
||
})
|
||
}
|
||
|
||
return &PacketResult{Packets: packets, Total: total}, nil
|
||
}
|
||
|
||
// parseDistinctIatasCSV turns SQLite GROUP_CONCAT output ("SJC,SFO,OAK") into
|
||
// a sorted, deduped []string. Returns an empty (non-nil) slice when the input
|
||
// is empty/nil so JSON serialization stays consistent (`[]` not `null`).
|
||
func parseDistinctIatasCSV(v interface{}) []string {
|
||
s, ok := v.(string)
|
||
if !ok || s == "" {
|
||
return []string{}
|
||
}
|
||
parts := strings.Split(s, ",")
|
||
seen := make(map[string]bool, len(parts))
|
||
out := make([]string, 0, len(parts))
|
||
for _, p := range parts {
|
||
code := strings.TrimSpace(p)
|
||
if code == "" || seen[code] {
|
||
continue
|
||
}
|
||
seen[code] = true
|
||
out = append(out, code)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) {
|
||
var where []string
|
||
var args []interface{}
|
||
|
||
if q.Type != nil {
|
||
where = append(where, "payload_type = ?")
|
||
args = append(args, *q.Type)
|
||
}
|
||
if q.Route != nil {
|
||
where = append(where, "route_type = ?")
|
||
args = append(args, *q.Route)
|
||
}
|
||
if q.Observer != "" {
|
||
where = append(where, "observer_id = ?")
|
||
args = append(args, q.Observer)
|
||
}
|
||
if q.Hash != "" {
|
||
where = append(where, "hash = ?")
|
||
args = append(args, strings.ToLower(q.Hash))
|
||
}
|
||
if q.Since != "" {
|
||
where = append(where, "timestamp > ?")
|
||
args = append(args, q.Since)
|
||
}
|
||
if q.Until != "" {
|
||
where = append(where, "timestamp < ?")
|
||
args = append(args, q.Until)
|
||
}
|
||
if q.Region != "" {
|
||
where = append(where, "observer_id IN (SELECT id FROM observers WHERE iata = ?)")
|
||
args = append(args, q.Region)
|
||
}
|
||
if q.Node != "" {
|
||
pk := db.resolveNodePubkey(q.Node)
|
||
// #1143: exact-match on the dedicated from_pubkey column instead of
|
||
// LIKE-on-JSON substring (adversarial spoof + same-name false positives).
|
||
where = append(where, "from_pubkey = ?")
|
||
args = append(args, pk)
|
||
}
|
||
return where, args
|
||
}
|
||
|
||
// buildTransmissionWhere builds WHERE clauses for transmission-centric queries.
|
||
// Uses t. prefix for transmission columns and EXISTS subqueries for observation filters.
|
||
func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) {
|
||
var where []string
|
||
var args []interface{}
|
||
|
||
if q.Type != nil {
|
||
where = append(where, "t.payload_type = ?")
|
||
args = append(args, *q.Type)
|
||
}
|
||
if q.Route != nil {
|
||
where = append(where, "t.route_type = ?")
|
||
args = append(args, *q.Route)
|
||
}
|
||
if q.Hash != "" {
|
||
where = append(where, "t.hash = ?")
|
||
args = append(args, strings.ToLower(q.Hash))
|
||
}
|
||
if q.Since != "" {
|
||
// RFC3339 since/until use an observations.timestamp subquery so that
|
||
// re-observed packets (whose t.first_seen is older than the window
|
||
// but which have observations inside the window) are still included.
|
||
// Non-RFC3339 falls back to t.first_seen string compare.
|
||
if ts, err := time.Parse(time.RFC3339Nano, q.Since); err == nil {
|
||
where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= ?)")
|
||
args = append(args, ts.Unix())
|
||
} else {
|
||
where = append(where, "t.first_seen > ?")
|
||
args = append(args, q.Since)
|
||
}
|
||
}
|
||
if q.Until != "" {
|
||
if ts, err := time.Parse(time.RFC3339Nano, q.Until); err == nil {
|
||
where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= ?)")
|
||
args = append(args, ts.Unix())
|
||
} else {
|
||
where = append(where, "t.first_seen < ?")
|
||
args = append(args, q.Until)
|
||
}
|
||
}
|
||
if q.Node != "" {
|
||
pk := db.resolveNodePubkey(q.Node)
|
||
// #1143: exact-match on dedicated from_pubkey column.
|
||
where = append(where, "t.from_pubkey = ?")
|
||
args = append(args, pk)
|
||
}
|
||
if q.Channel != "" {
|
||
// channel_hash column is indexed for payload_type = 5; filter is exact match.
|
||
where = append(where, "t.channel_hash = ?")
|
||
args = append(args, q.Channel)
|
||
}
|
||
if q.Observer != "" {
|
||
ids := strings.Split(q.Observer, ",")
|
||
placeholders := strings.Repeat("?,", len(ids))
|
||
placeholders = placeholders[:len(placeholders)-1]
|
||
if db.isV3 {
|
||
where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id IN ("+placeholders+"))")
|
||
} else {
|
||
where = append(where, "EXISTS (SELECT 1 FROM observations oi WHERE oi.transmission_id = t.id AND oi.observer_id IN ("+placeholders+"))")
|
||
}
|
||
for _, id := range ids {
|
||
args = append(args, strings.TrimSpace(id))
|
||
}
|
||
}
|
||
if q.Region != "" {
|
||
if db.isV3 {
|
||
where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = ?)")
|
||
} else {
|
||
where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = ?)")
|
||
}
|
||
args = append(args, q.Region)
|
||
}
|
||
return where, args
|
||
}
|
||
|
||
func (db *DB) resolveNodePubkey(nodeIDOrName string) string {
|
||
var pk string
|
||
err := db.conn.QueryRow("SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk)
|
||
if err != nil {
|
||
return nodeIDOrName
|
||
}
|
||
return pk
|
||
}
|
||
|
||
// GetTransmissionByID fetches from transmissions table with observer data.
|
||
func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) {
|
||
selectCols, observerJoin := db.transmissionBaseSQL()
|
||
querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = ?", selectCols, observerJoin)
|
||
|
||
rows, err := db.conn.Query(querySQL, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
if rows.Next() {
|
||
return db.scanTransmissionRow(rows), nil
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
// GetPacketByHash fetches a transmission by content hash with observer data.
|
||
func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) {
|
||
selectCols, observerJoin := db.transmissionBaseSQL()
|
||
querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = ?", selectCols, observerJoin)
|
||
|
||
rows, err := db.conn.Query(querySQL, strings.ToLower(hash))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
if rows.Next() {
|
||
return db.scanTransmissionRow(rows), nil
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
// GetObservationsForHash returns all observations for the transmission with
|
||
// the given content hash. Used as a fallback by the packet-detail handler
|
||
// when the in-memory PacketStore has pruned the entry but the DB still has it.
|
||
func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} {
|
||
var txID int
|
||
err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?",
|
||
strings.ToLower(hash)).Scan(&txID)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
obsByTx := db.getObservationsForTransmissions([]int{txID})
|
||
return obsByTx[txID]
|
||
}
|
||
|
||
// GetNodes returns filtered, paginated node list.
|
||
func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortBy, region string) ([]map[string]interface{}, int, map[string]int, error) {
|
||
var where []string
|
||
var args []interface{}
|
||
|
||
if role != "" {
|
||
where = append(where, "role = ?")
|
||
args = append(args, role)
|
||
}
|
||
if search != "" {
|
||
where = append(where, "name LIKE ?")
|
||
args = append(args, "%"+search+"%")
|
||
}
|
||
if before != "" {
|
||
where = append(where, "first_seen <= ?")
|
||
args = append(args, before)
|
||
}
|
||
if lastHeard != "" {
|
||
durations := map[string]int64{
|
||
"1h": 3600000, "6h": 21600000, "24h": 86400000,
|
||
"7d": 604800000, "30d": 2592000000,
|
||
}
|
||
if ms, ok := durations[lastHeard]; ok {
|
||
since := time.Now().Add(-time.Duration(ms) * time.Millisecond).Format(time.RFC3339)
|
||
where = append(where, "last_seen > ?")
|
||
args = append(args, since)
|
||
}
|
||
}
|
||
|
||
if region != "" {
|
||
codes := normalizeRegionCodes(region)
|
||
if len(codes) > 0 {
|
||
placeholders := make([]string, len(codes))
|
||
regionArgs := make([]interface{}, len(codes))
|
||
for i, c := range codes {
|
||
placeholders[i] = "?"
|
||
regionArgs[i] = c
|
||
}
|
||
joinCond := "obs.rowid = o.observer_idx"
|
||
if !db.isV3 {
|
||
joinCond = "obs.id = o.observer_id"
|
||
}
|
||
subq := fmt.Sprintf(`public_key IN (
|
||
SELECT DISTINCT JSON_EXTRACT(t.decoded_json, '$.pubKey')
|
||
FROM transmissions t
|
||
JOIN observations o ON o.transmission_id = t.id
|
||
JOIN observers obs ON %s
|
||
WHERE t.payload_type = 4
|
||
AND UPPER(TRIM(obs.iata)) IN (%s)
|
||
)`, joinCond, strings.Join(placeholders, ","))
|
||
where = append(where, subq)
|
||
args = append(args, regionArgs...)
|
||
}
|
||
}
|
||
|
||
w := ""
|
||
if len(where) > 0 {
|
||
w = "WHERE " + strings.Join(where, " AND ")
|
||
}
|
||
|
||
sortMap := map[string]string{
|
||
"name": "name ASC", "lastSeen": "last_seen DESC", "packetCount": "advert_count DESC",
|
||
}
|
||
order := "last_seen DESC"
|
||
if s, ok := sortMap[sortBy]; ok {
|
||
order = s
|
||
}
|
||
|
||
if limit <= 0 {
|
||
limit = 50
|
||
}
|
||
|
||
var total int
|
||
db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM nodes %s", w), args...).Scan(&total)
|
||
|
||
querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT ? OFFSET ?", db.nodeSelectCols(), w, order)
|
||
qArgs := append(args, limit, offset)
|
||
|
||
rows, err := db.conn.Query(querySQL, qArgs...)
|
||
if err != nil {
|
||
return nil, 0, nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
nodes := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
n := db.scanNodeRow(rows)
|
||
if n != nil {
|
||
nodes = append(nodes, n)
|
||
}
|
||
}
|
||
|
||
counts := db.GetAllRoleCounts()
|
||
return nodes, total, counts, nil
|
||
}
|
||
|
||
// SearchNodes searches nodes by name or pubkey prefix.
|
||
func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, error) {
|
||
if limit <= 0 {
|
||
limit = 10
|
||
}
|
||
rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name LIKE ? OR public_key LIKE ? ORDER BY last_seen DESC LIMIT ?", db.nodeSelectCols()),
|
||
"%"+query+"%", query+"%", limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
nodes := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
n := db.scanNodeRow(rows)
|
||
if n != nil {
|
||
nodes = append(nodes, n)
|
||
}
|
||
}
|
||
return nodes, nil
|
||
}
|
||
|
||
// GetNodeByPrefix resolves a hex prefix (>=8 chars) to a unique node.
|
||
// Returns (node, ambiguous, error). When multiple nodes share the prefix,
|
||
// returns (nil, true, nil). Used by the short-URL feature (issue #772).
|
||
//
|
||
// Trade-off vs an opaque ID lookup table: prefixes are stable across
|
||
// restarts, self-describing (no allocator needed), and resolve to the
|
||
// authoritative pubkey on the server. Cost: ambiguity grows with the
|
||
// node directory; we mitigate with a hard 8-hex-char (32-bit) minimum
|
||
// and surface 409 Conflict when collisions occur.
|
||
func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, error) {
|
||
if len(prefix) < 8 {
|
||
return nil, false, nil
|
||
}
|
||
// Validate hex (avoid SQL LIKE wildcards leaking through).
|
||
for _, c := range prefix {
|
||
isHex := (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
|
||
if !isHex {
|
||
return nil, false, nil
|
||
}
|
||
}
|
||
rows, err := db.conn.Query(
|
||
fmt.Sprintf("SELECT %s FROM nodes WHERE public_key LIKE ? LIMIT 2", db.nodeSelectCols()),
|
||
prefix+"%",
|
||
)
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
defer rows.Close()
|
||
var first map[string]interface{}
|
||
count := 0
|
||
for rows.Next() {
|
||
n := db.scanNodeRow(rows)
|
||
if n == nil {
|
||
continue
|
||
}
|
||
count++
|
||
if count == 1 {
|
||
first = n
|
||
} else {
|
||
return nil, true, nil
|
||
}
|
||
}
|
||
if count == 0 {
|
||
return nil, false, nil
|
||
}
|
||
return first, false, nil
|
||
}
|
||
|
||
// GetNodeByPubkey returns a single node.
|
||
func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) {
|
||
rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = ?", db.nodeSelectCols()), pubkey)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
if rows.Next() {
|
||
return db.scanNodeRow(rows), nil
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
// GetRecentTransmissionsForNode returns recent transmissions originated by a
|
||
// node, identified by exact pubkey match on the indexed from_pubkey column
|
||
// (#1143). The legacy `name` substring fallback was removed: it produced
|
||
// same-name false positives and an adversarial spoof path where any node
|
||
// could attribute its transmissions to a victim by naming itself with the
|
||
// victim's pubkey. Pubkey is unique by design — that's the whole point.
|
||
func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[string]interface{}, error) {
|
||
if limit <= 0 {
|
||
limit = 20
|
||
}
|
||
|
||
selectCols, observerJoin := db.transmissionBaseSQL()
|
||
|
||
// #1345: order by ingest id, not first_seen (=rxTime). Buffered observer
|
||
// uploads with old rxTime would otherwise displace fresh activity from
|
||
// the "recent transmissions for node" list.
|
||
querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = ? ORDER BY t.id DESC LIMIT ?",
|
||
selectCols, observerJoin)
|
||
args := []interface{}{pubkey, limit}
|
||
|
||
rows, err := db.conn.Query(querySQL, args...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
packets := make([]map[string]interface{}, 0)
|
||
var txIDs []int
|
||
for rows.Next() {
|
||
p := db.scanTransmissionRow(rows)
|
||
if p != nil {
|
||
// Placeholder for observations — filled below
|
||
p["observations"] = []map[string]interface{}{}
|
||
if id, ok := p["id"].(int); ok {
|
||
txIDs = append(txIDs, id)
|
||
}
|
||
packets = append(packets, p)
|
||
}
|
||
}
|
||
|
||
// Fetch observations for all transmissions
|
||
if len(txIDs) > 0 {
|
||
obsMap := db.getObservationsForTransmissions(txIDs)
|
||
for _, p := range packets {
|
||
if id, ok := p["id"].(int); ok {
|
||
if obs, found := obsMap[id]; found {
|
||
p["observations"] = obs
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return packets, nil
|
||
}
|
||
|
||
// getObservationsForTransmissions fetches all observations for a set of transmission IDs,
|
||
// returning a map of txID → []observation maps (matching Node.js recentAdverts shape).
|
||
func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string]interface{} {
|
||
result := make(map[int][]map[string]interface{})
|
||
if len(txIDs) == 0 {
|
||
return result
|
||
}
|
||
|
||
// Build IN clause
|
||
placeholders := make([]string, len(txIDs))
|
||
args := make([]interface{}, len(txIDs))
|
||
for i, id := range txIDs {
|
||
placeholders[i] = "?"
|
||
args[i] = id
|
||
}
|
||
|
||
var querySQL string
|
||
if db.isV3 {
|
||
querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata,
|
||
o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp
|
||
FROM observations o
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE o.transmission_id IN (%s)
|
||
ORDER BY o.timestamp DESC`, strings.Join(placeholders, ","))
|
||
} else {
|
||
querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, '') AS observer_iata,
|
||
o.direction, o.snr, o.rssi, o.path_json, o.timestamp AS obs_timestamp
|
||
FROM observations o
|
||
LEFT JOIN observers obs ON obs.id = o.observer_id
|
||
WHERE o.transmission_id IN (%s)
|
||
ORDER BY o.timestamp DESC`, strings.Join(placeholders, ","))
|
||
}
|
||
|
||
rows, err := db.conn.Query(querySQL, args...)
|
||
if err != nil {
|
||
return result
|
||
}
|
||
defer rows.Close()
|
||
|
||
for rows.Next() {
|
||
var txID, obsID int
|
||
var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString
|
||
var snr, rssi sql.NullFloat64
|
||
|
||
if err := rows.Scan(&txID, &obsID, &observerID, &observerName, &observerIATA, &direction,
|
||
&snr, &rssi, &pathJSON, &obsTimestamp); err != nil {
|
||
continue
|
||
}
|
||
|
||
ts := nullStr(obsTimestamp)
|
||
if s, ok := ts.(string); ok {
|
||
ts = normalizeTimestamp(s)
|
||
}
|
||
|
||
obs := map[string]interface{}{
|
||
"id": obsID,
|
||
"transmission_id": txID,
|
||
"observer_id": nullStr(observerID),
|
||
"observer_name": nullStr(observerName),
|
||
"observer_iata": nullStr(observerIATA),
|
||
"snr": nullFloat(snr),
|
||
"rssi": nullFloat(rssi),
|
||
"path_json": nullStr(pathJSON),
|
||
"timestamp": ts,
|
||
}
|
||
result[txID] = append(result[txID], obs)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
// GetObservers returns active observers (not soft-deleted) sorted by last_seen DESC.
|
||
func (db *DB) GetObservers() ([]Observer, error) {
|
||
// Issue #1290: can_relay is read via COALESCE(can_relay, 1). The
|
||
// column is added by internal/dbschema; older test fixtures and
|
||
// pre-migration DBs may lack it, so we probe and fall back.
|
||
// PR #1624 MAJOR-2: can_relay_seen is the tri-state sentinel — 1
|
||
// means the ingestor explicitly wrote a value, 0 means "unknown"
|
||
// and the server returns CanRelay=nil so the UI shows no badge.
|
||
canRelayClause := "COALESCE(can_relay, 1)"
|
||
canRelaySeenClause := "0"
|
||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol {
|
||
canRelayClause = "1"
|
||
}
|
||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol {
|
||
canRelaySeenClause = "COALESCE(can_relay_seen, 0)"
|
||
}
|
||
rows, err := db.conn.Query(`SELECT id, name, iata, last_seen, first_seen, packet_count,
|
||
model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at,
|
||
clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at,
|
||
` + canRelayClause + `, ` + canRelaySeenClause + `
|
||
FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
var observers []Observer
|
||
for rows.Next() {
|
||
var o Observer
|
||
var batteryMv, uptimeSecs, clockSkewSec sql.NullInt64
|
||
var clockSkewCount sql.NullInt64
|
||
var noiseFloor sql.NullFloat64
|
||
var canRelay, canRelaySeen int
|
||
if err := rows.Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount,
|
||
&o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt,
|
||
&clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen); err != nil {
|
||
continue
|
||
}
|
||
if canRelaySeen != 0 {
|
||
b := canRelay != 0
|
||
o.CanRelay = &b
|
||
}
|
||
if batteryMv.Valid {
|
||
v := int(batteryMv.Int64)
|
||
o.BatteryMv = &v
|
||
}
|
||
if uptimeSecs.Valid {
|
||
o.UptimeSecs = &uptimeSecs.Int64
|
||
}
|
||
if noiseFloor.Valid {
|
||
o.NoiseFloor = &noiseFloor.Float64
|
||
}
|
||
if clockSkewSec.Valid {
|
||
v := clockSkewSec.Int64
|
||
o.ClockSkewSeconds = &v
|
||
}
|
||
if clockSkewCount.Valid {
|
||
o.ClockSkewCount24h = int(clockSkewCount.Int64)
|
||
}
|
||
observers = append(observers, o)
|
||
}
|
||
return observers, nil
|
||
}
|
||
|
||
// GetNonRelayObserverPubkeys returns the lowercase observer.id pubkeys
|
||
// for observers that have advertised `repeat:off` (#1290). The server's
|
||
// path-hop disambiguator consumes this to exclude listener-only nodes
|
||
// from the candidate set. Inactive observers are excluded for
|
||
// consistency with GetObservers; reactivation flips can_relay only on
|
||
// the next status message.
|
||
func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) {
|
||
// Graceful no-op when can_relay column is absent (legacy DB / older
|
||
// test fixture). Avoids noisy schema-degradation log spam.
|
||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol {
|
||
return nil, nil
|
||
}
|
||
rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers
|
||
WHERE COALESCE(can_relay, 1) = 0
|
||
AND (inactive IS NULL OR inactive = 0)`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []string
|
||
for rows.Next() {
|
||
var pk string
|
||
if err := rows.Scan(&pk); err == nil && pk != "" {
|
||
out = append(out, pk)
|
||
}
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// GetCanRelaySeenObserverPubkeys returns the lowercase observer.id
|
||
// pubkeys for which the ingestor has explicitly written a repeat-field
|
||
// value (can_relay_seen=1). PR #1624 MAJOR-2: the badge surface uses
|
||
// this to render tri-state — observers NOT in this set are "unknown"
|
||
// and the UI shows no badge.
|
||
func (db *DB) GetCanRelaySeenObserverPubkeys() ([]string, error) {
|
||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); !hasCol {
|
||
return nil, nil
|
||
}
|
||
rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers
|
||
WHERE COALESCE(can_relay_seen, 0) = 1
|
||
AND (inactive IS NULL OR inactive = 0)`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []string
|
||
for rows.Next() {
|
||
var pk string
|
||
if err := rows.Scan(&pk); err == nil && pk != "" {
|
||
out = append(out, pk)
|
||
}
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// GetObserverByID returns a single observer.
|
||
func (db *DB) GetObserverByID(id string) (*Observer, error) {
|
||
var o Observer
|
||
var batteryMv, uptimeSecs, clockSkewSec sql.NullInt64
|
||
var clockSkewCount sql.NullInt64
|
||
var noiseFloor sql.NullFloat64
|
||
var canRelay, canRelaySeen int
|
||
canRelayClause := "COALESCE(can_relay, 1)"
|
||
canRelaySeenClause := "0"
|
||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol {
|
||
canRelayClause = "1"
|
||
}
|
||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol {
|
||
canRelaySeenClause = "COALESCE(can_relay_seen, 0)"
|
||
}
|
||
err := db.conn.QueryRow(`SELECT id, name, iata, last_seen, first_seen, packet_count,
|
||
model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at,
|
||
clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at,
|
||
`+canRelayClause+`, `+canRelaySeenClause+`
|
||
FROM observers WHERE id = ?`, id).
|
||
Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount,
|
||
&o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt,
|
||
&clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if canRelaySeen != 0 {
|
||
b := canRelay != 0
|
||
o.CanRelay = &b
|
||
}
|
||
if batteryMv.Valid {
|
||
v := int(batteryMv.Int64)
|
||
o.BatteryMv = &v
|
||
}
|
||
if uptimeSecs.Valid {
|
||
o.UptimeSecs = &uptimeSecs.Int64
|
||
}
|
||
if noiseFloor.Valid {
|
||
o.NoiseFloor = &noiseFloor.Float64
|
||
}
|
||
if clockSkewSec.Valid {
|
||
v := clockSkewSec.Int64
|
||
o.ClockSkewSeconds = &v
|
||
}
|
||
if clockSkewCount.Valid {
|
||
o.ClockSkewCount24h = int(clockSkewCount.Int64)
|
||
}
|
||
return &o, nil
|
||
}
|
||
|
||
// GetObserverIdsForRegion returns observer IDs for given IATA codes.
|
||
func (db *DB) GetObserverIdsForRegion(regionParam string) ([]string, error) {
|
||
codes := normalizeRegionCodes(regionParam)
|
||
if len(codes) == 0 {
|
||
return nil, nil
|
||
}
|
||
placeholders := make([]string, len(codes))
|
||
args := make([]interface{}, len(codes))
|
||
for i, c := range codes {
|
||
placeholders[i] = "?"
|
||
args[i] = c
|
||
}
|
||
rows, err := db.conn.Query(fmt.Sprintf("SELECT id FROM observers WHERE UPPER(TRIM(iata)) IN (%s)", strings.Join(placeholders, ",")), args...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var ids []string
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, err
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
return ids, nil
|
||
}
|
||
|
||
// normalizeRegionCodes parses a region query parameter into a list of upper-case
|
||
// IATA codes. Returns nil to signal "no filter" (match all regions).
|
||
//
|
||
// Sentinel handling (issue #770): the frontend region filter dropdown labels its
|
||
// catch-all option "All". When that option is selected the UI may send
|
||
// ?region=All; older code interpreted that literally and tried to match an
|
||
// IATA code "ALL", which never exists, returning an empty result set. Treat
|
||
// "All" / "ALL" / "all" (case-insensitive, optionally surrounded by whitespace
|
||
// or mixed with empty CSV slots) as equivalent to an empty value.
|
||
//
|
||
// Real IATA codes (e.g. "SJC", "PDX") still pass through unchanged.
|
||
func normalizeRegionCodes(regionParam string) []string {
|
||
if regionParam == "" {
|
||
return nil
|
||
}
|
||
tokens := strings.Split(regionParam, ",")
|
||
codes := make([]string, 0, len(tokens))
|
||
for _, token := range tokens {
|
||
code := strings.TrimSpace(strings.ToUpper(token))
|
||
if code == "" || code == "ALL" {
|
||
continue
|
||
}
|
||
codes = append(codes, code)
|
||
}
|
||
if len(codes) == 0 {
|
||
return nil
|
||
}
|
||
return codes
|
||
}
|
||
|
||
// GetDistinctIATAs returns all distinct IATA codes from observers.
|
||
func (db *DB) GetDistinctIATAs() ([]string, error) {
|
||
rows, err := db.conn.Query("SELECT DISTINCT iata FROM observers WHERE iata IS NOT NULL")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var codes []string
|
||
for rows.Next() {
|
||
var code string
|
||
rows.Scan(&code)
|
||
codes = append(codes, code)
|
||
}
|
||
return codes, nil
|
||
}
|
||
|
||
// GetNetworkStatus returns overall network health status.
|
||
func (db *DB) GetNetworkStatus(healthThresholds HealthThresholds) (map[string]interface{}, error) {
|
||
rows, err := db.conn.Query("SELECT public_key, name, role, last_seen FROM nodes")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
now := time.Now().UnixMilli()
|
||
active, degraded, silent, total := 0, 0, 0, 0
|
||
roleCounts := map[string]int{}
|
||
|
||
for rows.Next() {
|
||
var pk string
|
||
var name, role, lastSeen sql.NullString
|
||
rows.Scan(&pk, &name, &role, &lastSeen)
|
||
total++
|
||
r := "unknown"
|
||
if role.Valid {
|
||
r = role.String
|
||
}
|
||
roleCounts[r]++
|
||
|
||
age := int64(math.MaxInt64)
|
||
if lastSeen.Valid {
|
||
if t, err := time.Parse(time.RFC3339, lastSeen.String); err == nil {
|
||
age = now - t.UnixMilli()
|
||
} else if t, err := time.Parse("2006-01-02 15:04:05", lastSeen.String); err == nil {
|
||
age = now - t.UnixMilli()
|
||
}
|
||
}
|
||
degradedMs, silentMs := healthThresholds.GetHealthMs(r)
|
||
if age < int64(degradedMs) {
|
||
active++
|
||
} else if age < int64(silentMs) {
|
||
degraded++
|
||
} else {
|
||
silent++
|
||
}
|
||
}
|
||
|
||
return map[string]interface{}{
|
||
"total": total, "active": active, "degraded": degraded, "silent": silent,
|
||
"roleCounts": roleCounts,
|
||
}, nil
|
||
}
|
||
|
||
// GetTraces returns observations for a hash using direct table queries.
|
||
func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) {
|
||
var querySQL string
|
||
if db.isV3 {
|
||
querySQL = `SELECT obs.id AS observer_id, obs.name AS observer_name,
|
||
strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp,
|
||
o.snr, o.rssi, o.path_json
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.hash = ?
|
||
ORDER BY o.timestamp ASC`
|
||
} else {
|
||
querySQL = `SELECT o.observer_id, o.observer_name,
|
||
strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp,
|
||
o.snr, o.rssi, o.path_json
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
WHERE t.hash = ?
|
||
ORDER BY o.timestamp ASC`
|
||
}
|
||
rows, err := db.conn.Query(querySQL, strings.ToLower(hash))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var traces []map[string]interface{}
|
||
for rows.Next() {
|
||
var obsID, obsName, ts, pathJSON sql.NullString
|
||
var snr, rssi sql.NullFloat64
|
||
rows.Scan(&obsID, &obsName, &ts, &snr, &rssi, &pathJSON)
|
||
traces = append(traces, map[string]interface{}{
|
||
"observer": nullStr(obsID),
|
||
"observer_name": nullStr(obsName),
|
||
"time": nullStr(ts),
|
||
"snr": nullFloat(snr),
|
||
"rssi": nullFloat(rssi),
|
||
"path_json": nullStr(pathJSON),
|
||
})
|
||
}
|
||
if traces == nil {
|
||
traces = make([]map[string]interface{}, 0)
|
||
}
|
||
return traces, nil
|
||
}
|
||
|
||
// PacketPathPoint is one hop's position along a packet's resolved relay
|
||
// path, for map visualization (public/packet-path-map.js). Lat/Lon are
|
||
// nil when that node has never advertised a GPS position -- the caller
|
||
// draws a gap rather than guessing.
|
||
type PacketPathPoint struct {
|
||
PublicKey string `json:"publicKey"`
|
||
Name string `json:"name"`
|
||
Role string `json:"role,omitempty"`
|
||
Lat *float64 `json:"lat"`
|
||
Lon *float64 `json:"lon"`
|
||
// Approx is true when Lat/Lon are a weighted centroid of this node's
|
||
// positioned neighbor_edges neighbors rather than its own position
|
||
// (used as a last-resort stand-in when the node itself has no known
|
||
// fix). ApproxNeighborCount/ApproxSpreadKm are only meaningful when
|
||
// Approx is true.
|
||
Approx bool `json:"approx,omitempty"`
|
||
// ApproxNeighborCount is how many positioned neighbors fed the
|
||
// centroid -- a rough confidence signal, higher is more confident.
|
||
ApproxNeighborCount int `json:"approxNeighborCount,omitempty"`
|
||
// ApproxSpreadKm is the widest distance (km) between any two of
|
||
// those neighbors -- 0 (and omitted) with a single contributor;
|
||
// larger means the neighbors disagree more about where "nearby" is.
|
||
ApproxSpreadKm *float64 `json:"approxSpreadKm,omitempty"`
|
||
}
|
||
|
||
// PacketPathObserver is the station that produced a given branch's
|
||
// observation of a packet (see GetPacketPath), positioned from its own
|
||
// self-advertised GPS (the same source /api/observers uses) when known,
|
||
// falling back to its configured IATA code, and finally its strongest
|
||
// neighbor_edges neighbor's position (Approx=true), otherwise -- not a
|
||
// stored per-observer lat/lon column.
|
||
type PacketPathObserver struct {
|
||
// PublicKey is the observer's mesh pubkey (observers.id for v3,
|
||
// observer_id for legacy), when it has one -- lets callers link out
|
||
// to the node detail page. Empty for an observer whose id never
|
||
// resembled a pubkey.
|
||
PublicKey string `json:"publicKey,omitempty"`
|
||
Name string `json:"name"`
|
||
IATA string `json:"iata,omitempty"`
|
||
Role string `json:"role,omitempty"`
|
||
Lat *float64 `json:"lat"`
|
||
Lon *float64 `json:"lon"`
|
||
// Approx is true when Lat/Lon are a weighted centroid of this
|
||
// station's positioned neighbors instead of its own position -- see
|
||
// PacketPathPoint.Approx (and ApproxNeighborCount/ApproxSpreadKm).
|
||
Approx bool `json:"approx,omitempty"`
|
||
ApproxNeighborCount int `json:"approxNeighborCount,omitempty"`
|
||
ApproxSpreadKm *float64 `json:"approxSpreadKm,omitempty"`
|
||
}
|
||
|
||
// PacketPathBranch is one station's route to a packet: how far it
|
||
// traveled to reach them (hop count taken straight from that
|
||
// observation's path_json, independent of how much of it resolved) and,
|
||
// where resolvable, each hop's name/role/lat/lon in path order.
|
||
type PacketPathBranch struct {
|
||
Hops int `json:"hops"`
|
||
Points []PacketPathPoint `json:"points"`
|
||
Observer *PacketPathObserver `json:"observer,omitempty"`
|
||
SNR *float64 `json:"snr,omitempty"`
|
||
// SecondsAfterFirst is how long after the earliest-arriving
|
||
// observation (see PacketPathResponse.First) this branch's own
|
||
// deepest observation arrived, in seconds. Zero for First itself.
|
||
// Omitted when either timestamp is unknown.
|
||
SecondsAfterFirst *float64 `json:"secondsAfterFirst,omitempty"`
|
||
// DistanceFromFirstKm is the great-circle distance between this
|
||
// branch's own Observer and First's Observer. Zero for First itself.
|
||
// Omitted when either station's position is unknown (including when
|
||
// one or both are Approx -- an estimate compounding another estimate
|
||
// isn't worth surfacing).
|
||
DistanceFromFirstKm *float64 `json:"distanceFromFirstKm,omitempty"`
|
||
}
|
||
|
||
// TouchedAreaShape is one touched area's display label plus its
|
||
// configured boundary -- a real Polygon when the area was drawn as one,
|
||
// otherwise a bounding box (LatMin/LatMax/LonMin/LonMax), matching
|
||
// whichever AreaEntry (cmd/server/config.go) was configured with. Exactly
|
||
// one of Polygon or the LatMin.../LonMax... quartet is populated.
|
||
type TouchedAreaShape struct {
|
||
Label string `json:"label"`
|
||
Polygon [][2]float64 `json:"polygon,omitempty"`
|
||
LatMin *float64 `json:"latMin,omitempty"`
|
||
LatMax *float64 `json:"latMax,omitempty"`
|
||
LonMin *float64 `json:"lonMin,omitempty"`
|
||
LonMax *float64 `json:"lonMax,omitempty"`
|
||
}
|
||
|
||
// PacketPathResponse is every branch a packet is known to have reached --
|
||
// one per distinct observer, kept at that observer's own deepest
|
||
// observation -- used to draw the full flood spread on a map (the
|
||
// ping-bot reply's "View path" link), not just the single farthest route.
|
||
// First is the single EARLIEST-arriving observation across every station
|
||
// (regardless of observer or hop depth) -- the same "first observation
|
||
// wins" pick already used for a ping message's own meta line -- included
|
||
// separately since it approximates where the message entered the visible
|
||
// mesh, a landmark the deepest-first Branches ordering doesn't surface.
|
||
type PacketPathResponse struct {
|
||
Hash string `json:"hash"`
|
||
Branches []PacketPathBranch `json:"branches"`
|
||
First *PacketPathBranch `json:"first,omitempty"`
|
||
// TouchedAreas is every configured area any point or observer on this
|
||
// path resolved to (deduped, alphabetized, uncapped -- unlike the
|
||
// ping-bot reply's capped list, the map has room to show all of them),
|
||
// each carrying its configured boundary so the map can shade it, not
|
||
// just list it as text. Populated by routes.go's
|
||
// annotatePacketPathTouchedAreas, not here: area resolution needs
|
||
// config.Areas, not available at this SQL-only DB layer.
|
||
TouchedAreas []TouchedAreaShape `json:"touchedAreas,omitempty"`
|
||
}
|
||
|
||
// GetPacketPath resolves every distinct station that observed a packet to
|
||
// its own branch: hop count and (where resolvable) relay names/positions
|
||
// in path order, plus that station's own position. A station can hear a
|
||
// packet more than once as flood copies arrive via different routes; only
|
||
// its deepest observation (by raw hop count, same "farthest leg"
|
||
// reasoning as the ping-bot reply -- see pingBotReply's doc comment) is
|
||
// kept, so each station contributes exactly one branch. Branches are
|
||
// returned deepest-first. The single earliest-arriving observation
|
||
// (usually 0 hops, close to the sender) is additionally surfaced via
|
||
// First, independent of which station it came from or how deep it was.
|
||
func (db *DB) GetPacketPath(hash string) (*PacketPathResponse, error) {
|
||
if !db.hasResolvedPath {
|
||
return nil, fmt.Errorf("resolved_path not available on this server")
|
||
}
|
||
var querySQL string
|
||
if db.isV3 {
|
||
querySQL = `SELECT obs.rowid, obs.id, obs.name, obs.iata, o.path_json, o.resolved_path, o.snr, o.timestamp
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.hash = ?`
|
||
} else {
|
||
querySQL = `SELECT o.observer_id, o.observer_id, o.observer_name, NULL, o.path_json, o.resolved_path, o.snr, o.timestamp
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
WHERE t.hash = ?`
|
||
}
|
||
rows, err := db.conn.Query(querySQL, strings.ToLower(hash))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("packet path query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
type obsBranch struct {
|
||
hops int
|
||
resolvedPath []*string
|
||
observerName string
|
||
observerPubkey string
|
||
observerIATA sql.NullString
|
||
snr sql.NullFloat64
|
||
ts int64 // unix epoch seconds of the observation that produced this branch's hops/resolvedPath; 0 if unknown
|
||
}
|
||
best := make(map[string]*obsBranch)
|
||
var first *obsBranch
|
||
var firstTS int64
|
||
|
||
for rows.Next() {
|
||
var obsKey, obsPubkey, obsName, obsIATA, pathJSON, resolvedPathJSON sql.NullString
|
||
var snr sql.NullFloat64
|
||
var ts sql.NullInt64
|
||
if err := rows.Scan(&obsKey, &obsPubkey, &obsName, &obsIATA, &pathJSON, &resolvedPathJSON, &snr, &ts); err != nil {
|
||
continue
|
||
}
|
||
if !pathJSON.Valid {
|
||
continue
|
||
}
|
||
var h []string
|
||
if json.Unmarshal([]byte(pathJSON.String), &h) != nil {
|
||
continue
|
||
}
|
||
hops := len(h)
|
||
var resolvedPath []*string
|
||
if resolvedPathJSON.Valid {
|
||
resolvedPath = unmarshalResolvedPath(resolvedPathJSON.String)
|
||
}
|
||
key := obsKey.String
|
||
if key == "" {
|
||
key = obsName.String
|
||
}
|
||
if key == "" {
|
||
continue // no way to attribute this observation to a station
|
||
}
|
||
var tsVal int64
|
||
if ts.Valid {
|
||
tsVal = ts.Int64
|
||
}
|
||
branch := &obsBranch{
|
||
hops: hops, resolvedPath: resolvedPath,
|
||
observerName: obsName.String, observerPubkey: strings.ToLower(strings.TrimSpace(obsPubkey.String)),
|
||
observerIATA: obsIATA, snr: snr, ts: tsVal,
|
||
}
|
||
if existing, ok := best[key]; !ok || hops > existing.hops {
|
||
best[key] = branch
|
||
}
|
||
if ts.Valid && (first == nil || ts.Int64 < firstTS) {
|
||
first, firstTS = branch, ts.Int64
|
||
}
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, fmt.Errorf("packet path iteration: %w", err)
|
||
}
|
||
|
||
resp := &PacketPathResponse{Hash: hash, Branches: []PacketPathBranch{}}
|
||
if len(best) == 0 {
|
||
return resp, nil
|
||
}
|
||
|
||
pubkeySet := map[string]bool{}
|
||
if first != nil {
|
||
for _, pk := range first.resolvedPath {
|
||
if pk != nil && *pk != "" {
|
||
pubkeySet[*pk] = true
|
||
}
|
||
}
|
||
if first.observerPubkey != "" {
|
||
pubkeySet[first.observerPubkey] = true
|
||
}
|
||
}
|
||
for _, b := range best {
|
||
for _, pk := range b.resolvedPath {
|
||
if pk != nil && *pk != "" {
|
||
pubkeySet[*pk] = true
|
||
}
|
||
}
|
||
// An observer is itself a mesh node -- if it has ever self-advertised
|
||
// a GPS position, that's a more precise fix than its (often manually
|
||
// typed, sometimes wrong or missing) configured IATA code. Folded
|
||
// into the same batched lookup below rather than a second query.
|
||
if b.observerPubkey != "" {
|
||
pubkeySet[b.observerPubkey] = true
|
||
}
|
||
}
|
||
pubkeys := make([]string, 0, len(pubkeySet))
|
||
for pk := range pubkeySet {
|
||
pubkeys = append(pubkeys, pk)
|
||
}
|
||
type nodeInfo struct {
|
||
name string
|
||
role string
|
||
lat *float64
|
||
lon *float64
|
||
}
|
||
nodeByPK := make(map[string]nodeInfo, len(pubkeys))
|
||
if len(pubkeys) > 0 {
|
||
placeholders := make([]byte, 0, len(pubkeys)*2)
|
||
args := make([]interface{}, len(pubkeys))
|
||
for i, pk := range pubkeys {
|
||
if i > 0 {
|
||
placeholders = append(placeholders, ',')
|
||
}
|
||
placeholders = append(placeholders, '?')
|
||
args[i] = pk
|
||
}
|
||
nodeRows, err := db.conn.Query(
|
||
"SELECT public_key, name, role, lat, lon FROM nodes WHERE public_key IN ("+string(placeholders)+")", args...)
|
||
if err == nil {
|
||
for nodeRows.Next() {
|
||
var pk string
|
||
var name, role sql.NullString
|
||
var lat, lon sql.NullFloat64
|
||
if nodeRows.Scan(&pk, &name, &role, &lat, &lon) == nil {
|
||
ni := nodeInfo{name: name.String, role: role.String}
|
||
// (0,0) is the ocean off Ghana, not a real fix -- some
|
||
// nodes have it stored literally instead of NULL when
|
||
// they've never actually reported a GPS position.
|
||
// Excluded the same way GetNodesForScopeAdoption and
|
||
// geofilter.PassesFilter already do; the node's own
|
||
// name/role are kept, just not its bogus position.
|
||
if lat.Valid && lon.Valid && !(lat.Float64 == 0 && lon.Float64 == 0) {
|
||
v1, v2 := lat.Float64, lon.Float64
|
||
ni.lat, ni.lon = &v1, &v2
|
||
}
|
||
nodeByPK[pk] = ni
|
||
}
|
||
}
|
||
nodeRows.Close()
|
||
}
|
||
}
|
||
|
||
// Fallback for observers whose own `observers.id` isn't its mesh
|
||
// pubkey at all -- e.g. an MQTT-bridge-type observer (observed:
|
||
// openHop-Repeater firmware) that publishes its status keyed by
|
||
// device name rather than pubkey, so the lookup above never matches
|
||
// even though the same physical device has a real, positioned nodes
|
||
// row under its actual pubkey. Matched by exact display name; only
|
||
// queried for observers the pubkey lookup left unpositioned, and only
|
||
// applied if the pubkey lookup didn't already find something.
|
||
nameFallbackNeeded := map[string]bool{}
|
||
needsNameFallback := func(b *obsBranch) {
|
||
if b == nil || b.observerName == "" {
|
||
return
|
||
}
|
||
if ni, ok := nodeByPK[b.observerPubkey]; ok && ni.lat != nil && ni.lon != nil {
|
||
return
|
||
}
|
||
nameFallbackNeeded[b.observerName] = true
|
||
}
|
||
for _, b := range best {
|
||
needsNameFallback(b)
|
||
}
|
||
needsNameFallback(first)
|
||
nodeByName := make(map[string]nodeInfo, len(nameFallbackNeeded))
|
||
if len(nameFallbackNeeded) > 0 {
|
||
names := make([]string, 0, len(nameFallbackNeeded))
|
||
for n := range nameFallbackNeeded {
|
||
names = append(names, n)
|
||
}
|
||
placeholders := make([]byte, 0, len(names)*2)
|
||
args := make([]interface{}, len(names))
|
||
for i, n := range names {
|
||
if i > 0 {
|
||
placeholders = append(placeholders, ',')
|
||
}
|
||
placeholders = append(placeholders, '?')
|
||
args[i] = n
|
||
}
|
||
nameRows, err := db.conn.Query(
|
||
"SELECT name, role, lat, lon FROM nodes WHERE name IN ("+string(placeholders)+") AND lat IS NOT NULL AND lon IS NOT NULL AND lat != 0 AND lon != 0", args...)
|
||
if err == nil {
|
||
ambiguous := map[string]bool{}
|
||
for nameRows.Next() {
|
||
var name, role sql.NullString
|
||
var lat, lon sql.NullFloat64
|
||
if nameRows.Scan(&name, &role, &lat, &lon) == nil {
|
||
if _, exists := nodeByName[name.String]; exists {
|
||
ambiguous[name.String] = true // >1 positioned node shares this name -- don't guess which one
|
||
continue
|
||
}
|
||
v1, v2 := lat.Float64, lon.Float64
|
||
nodeByName[name.String] = nodeInfo{role: role.String, lat: &v1, lon: &v2}
|
||
}
|
||
}
|
||
nameRows.Close()
|
||
for n := range ambiguous {
|
||
delete(nodeByName, n)
|
||
}
|
||
}
|
||
}
|
||
|
||
buildBranch := func(b *obsBranch) PacketPathBranch {
|
||
branch := PacketPathBranch{Hops: b.hops, Points: []PacketPathPoint{}}
|
||
for _, pk := range b.resolvedPath {
|
||
if pk == nil || *pk == "" {
|
||
continue
|
||
}
|
||
ni := nodeByPK[*pk]
|
||
name := ni.name
|
||
if name == "" {
|
||
name = *pk
|
||
}
|
||
point := PacketPathPoint{PublicKey: *pk, Name: name, Role: ni.role, Lat: ni.lat, Lon: ni.lon}
|
||
if point.Lat == nil {
|
||
// Last resort: this node has never itself reported a
|
||
// position -- borrow a weighted centroid of its
|
||
// positioned neighbors instead, clearly flagged as
|
||
// approximate rather than a real fix.
|
||
if _, nLat, nLon, nCount, nSpread, ok := db.nearestPositionedNeighbor(*pk); ok {
|
||
lat, lon := nLat, nLon
|
||
point.Lat, point.Lon, point.Approx, point.ApproxNeighborCount = &lat, &lon, true, nCount
|
||
if nCount > 1 {
|
||
s := nSpread
|
||
point.ApproxSpreadKm = &s
|
||
}
|
||
}
|
||
}
|
||
branch.Points = append(branch.Points, point)
|
||
}
|
||
if b.observerName != "" {
|
||
obs := &PacketPathObserver{Name: b.observerName, PublicKey: b.observerPubkey}
|
||
if b.observerIATA.Valid {
|
||
obs.IATA = strings.ToUpper(strings.TrimSpace(b.observerIATA.String))
|
||
}
|
||
// Prefer the observer's own self-advertised GPS (same source as
|
||
// /api/observers and the Wardriving tab), falling back to a
|
||
// name match against `nodes` (some bridge-type observers --
|
||
// seen on openHop-Repeater firmware -- publish their MQTT
|
||
// status keyed by device name rather than mesh pubkey, so the
|
||
// pubkey lookup above never finds their real, positioned node
|
||
// row), and only then the configured IATA code -- which only
|
||
// covers a fixed list of real airports plus a handful of
|
||
// hand-added local codes, so a custom/regional code an
|
||
// operator typed in (or a typo) falls through it even when
|
||
// the node itself knows exactly where it is.
|
||
if ni, ok := nodeByPK[b.observerPubkey]; ok && ni.role != "" {
|
||
obs.Role = ni.role
|
||
} else if ni, ok := nodeByName[b.observerName]; ok && ni.role != "" {
|
||
obs.Role = ni.role
|
||
}
|
||
if ni, ok := nodeByPK[b.observerPubkey]; ok && ni.lat != nil && ni.lon != nil {
|
||
obs.Lat, obs.Lon = ni.lat, ni.lon
|
||
} else if ni, ok := nodeByName[b.observerName]; ok {
|
||
obs.Lat, obs.Lon = ni.lat, ni.lon
|
||
} else if obs.IATA != "" {
|
||
if coord, ok := iataCoords[obs.IATA]; ok {
|
||
lat, lon := coord.Lat, coord.Lon
|
||
obs.Lat, obs.Lon = &lat, &lon
|
||
}
|
||
}
|
||
if obs.Lat == nil && b.observerPubkey != "" {
|
||
// Last resort, same as the hop-point fallback above: no
|
||
// position of its own anywhere, so borrow a weighted
|
||
// centroid of its positioned neighbors, flagged as approximate.
|
||
if _, nLat, nLon, nCount, nSpread, ok := db.nearestPositionedNeighbor(b.observerPubkey); ok {
|
||
lat, lon := nLat, nLon
|
||
obs.Lat, obs.Lon, obs.Approx, obs.ApproxNeighborCount = &lat, &lon, true, nCount
|
||
if nCount > 1 {
|
||
s := nSpread
|
||
obs.ApproxSpreadKm = &s
|
||
}
|
||
}
|
||
}
|
||
branch.Observer = obs
|
||
}
|
||
if b.snr.Valid {
|
||
v := b.snr.Float64
|
||
branch.SNR = &v
|
||
}
|
||
if b.ts > 0 && first != nil && first.ts > 0 {
|
||
d := float64(b.ts - first.ts)
|
||
branch.SecondsAfterFirst = &d
|
||
}
|
||
return branch
|
||
}
|
||
|
||
// Build First first so its Observer position is known before computing
|
||
// every other branch's DistanceFromFirstKm against it.
|
||
var firstLat, firstLon *float64
|
||
if first != nil {
|
||
fb := buildBranch(first)
|
||
if fb.Observer != nil && !fb.Observer.Approx {
|
||
firstLat, firstLon = fb.Observer.Lat, fb.Observer.Lon
|
||
}
|
||
if firstLat != nil {
|
||
zero := 0.0
|
||
fb.DistanceFromFirstKm = &zero
|
||
}
|
||
resp.First = &fb
|
||
}
|
||
|
||
for _, b := range best {
|
||
branch := buildBranch(b)
|
||
if firstLat != nil && branch.Observer != nil && !branch.Observer.Approx && branch.Observer.Lat != nil {
|
||
d := haversineKm(*branch.Observer.Lat, *branch.Observer.Lon, *firstLat, *firstLon)
|
||
branch.DistanceFromFirstKm = &d
|
||
}
|
||
resp.Branches = append(resp.Branches, branch)
|
||
}
|
||
sort.Slice(resp.Branches, func(i, j int) bool { return resp.Branches[i].Hops > resp.Branches[j].Hops })
|
||
|
||
return resp, nil
|
||
}
|
||
|
||
// nearestPositionedNeighbor estimates pubkey's position from its
|
||
// neighbor_edges neighbors that themselves have a real, known position,
|
||
// for use as an approximate stand-in when pubkey has none of its own --
|
||
// e.g. a node that's never advertised a GPS fix, but is almost
|
||
// certainly physically near wherever its neighbors are. Weighted by
|
||
// each edge's observation count (a neighbor seen relaying to/from
|
||
// pubkey many times pulls the estimate harder than one seen once), so
|
||
// with several positioned neighbors this settles somewhere among them
|
||
// rather than collapsing onto a single neighbor's exact coordinates --
|
||
// each neighbor's OWN position is a real, precise fix; only pubkey's
|
||
// position relative to them is unknown, so more of them narrows it
|
||
// down. With exactly one positioned neighbor this is identical to
|
||
// using that neighbor's position outright.
|
||
//
|
||
// contributorCount is how many positioned neighbors fed the estimate,
|
||
// and spreadKm is the widest distance between any two of them (0 when
|
||
// there's only one) -- together a rough confidence signal callers can
|
||
// use to size an "uncertainty" marker: more contributors that broadly
|
||
// agree (small spread) means a tighter estimate than a single neighbor
|
||
// or several that disagree (large spread).
|
||
//
|
||
// Returns ok=false when pubkey has no neighbor with a position at all.
|
||
func (db *DB) nearestPositionedNeighbor(pubkey string) (name string, lat, lon float64, contributorCount int, spreadKm float64, ok bool) {
|
||
pk := strings.ToLower(strings.TrimSpace(pubkey))
|
||
if pk == "" {
|
||
return "", 0, 0, 0, 0, false
|
||
}
|
||
rows, err := db.conn.Query(`
|
||
SELECT CASE WHEN node_a = ? THEN node_b ELSE node_a END AS neighbor, count
|
||
FROM neighbor_edges
|
||
WHERE node_a = ? OR node_b = ?
|
||
ORDER BY count DESC
|
||
LIMIT 20`, pk, pk, pk)
|
||
if err != nil {
|
||
return "", 0, 0, 0, 0, false
|
||
}
|
||
type candidate struct {
|
||
pubkey string
|
||
weight float64
|
||
}
|
||
var candidates []candidate
|
||
for rows.Next() {
|
||
var neighborPK string
|
||
var count float64
|
||
if rows.Scan(&neighborPK, &count) == nil {
|
||
candidates = append(candidates, candidate{pubkey: neighborPK, weight: count})
|
||
}
|
||
}
|
||
rows.Close()
|
||
if len(candidates) == 0 {
|
||
return "", 0, 0, 0, 0, false
|
||
}
|
||
|
||
placeholders := make([]byte, 0, len(candidates)*2)
|
||
args := make([]interface{}, len(candidates))
|
||
for i, c := range candidates {
|
||
if i > 0 {
|
||
placeholders = append(placeholders, ',')
|
||
}
|
||
placeholders = append(placeholders, '?')
|
||
args[i] = c.pubkey
|
||
}
|
||
type posInfo struct {
|
||
name string
|
||
lat, lon float64
|
||
}
|
||
posByPK := make(map[string]posInfo, len(candidates))
|
||
nodeRows, err := db.conn.Query(
|
||
"SELECT public_key, name, lat, lon FROM nodes WHERE public_key IN ("+string(placeholders)+") AND lat IS NOT NULL AND lon IS NOT NULL AND lat != 0 AND lon != 0", args...)
|
||
if err == nil {
|
||
for nodeRows.Next() {
|
||
var candPK string
|
||
var candName sql.NullString
|
||
var candLat, candLon float64
|
||
if nodeRows.Scan(&candPK, &candName, &candLat, &candLon) == nil {
|
||
posByPK[candPK] = posInfo{name: candName.String, lat: candLat, lon: candLon}
|
||
}
|
||
}
|
||
nodeRows.Close()
|
||
}
|
||
|
||
// candidates is count-DESC ordered, so the first contributor found
|
||
// is the strongest -- used only for the returned name, which is
|
||
// otherwise cosmetic (current callers discard it).
|
||
var sumLat, sumLon, sumWeight float64
|
||
var strongestName string
|
||
var contributors []posInfo
|
||
for _, c := range candidates {
|
||
p, found := posByPK[c.pubkey]
|
||
if !found {
|
||
continue
|
||
}
|
||
w := c.weight
|
||
if w <= 0 {
|
||
w = 1
|
||
}
|
||
sumLat += p.lat * w
|
||
sumLon += p.lon * w
|
||
sumWeight += w
|
||
if strongestName == "" {
|
||
strongestName = p.name
|
||
}
|
||
contributors = append(contributors, p)
|
||
}
|
||
if sumWeight == 0 {
|
||
return "", 0, 0, 0, 0, false
|
||
}
|
||
var spread float64
|
||
for i := 0; i < len(contributors); i++ {
|
||
for j := i + 1; j < len(contributors); j++ {
|
||
d := haversineKm(contributors[i].lat, contributors[i].lon, contributors[j].lat, contributors[j].lon)
|
||
if d > spread {
|
||
spread = d
|
||
}
|
||
}
|
||
}
|
||
return strongestName, sumLat / sumWeight, sumLon / sumWeight, len(contributors), spread, true
|
||
}
|
||
|
||
// GetChannels returns channel list from GRP_TXT packets.
|
||
// Queries transmissions directly (not a VIEW) to avoid observation-level
|
||
// duplicates that could cause stale lastMessage when an older message has
|
||
// a later re-observation timestamp.
|
||
func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) {
|
||
regionParam := ""
|
||
if len(region) > 0 {
|
||
regionParam = region[0]
|
||
}
|
||
|
||
// Check cache (60s TTL)
|
||
db.channelsCacheMu.Lock()
|
||
if db.channelsCacheRes != nil && db.channelsCacheKey == regionParam && time.Now().Before(db.channelsCacheExp) {
|
||
res := db.channelsCacheRes
|
||
db.channelsCacheMu.Unlock()
|
||
return res, nil
|
||
}
|
||
db.channelsCacheMu.Unlock()
|
||
|
||
regionCodes := normalizeRegionCodes(regionParam)
|
||
|
||
var querySQL string
|
||
args := make([]interface{}, 0, len(regionCodes))
|
||
|
||
if len(regionCodes) > 0 {
|
||
placeholders := make([]string, len(regionCodes))
|
||
for i, code := range regionCodes {
|
||
placeholders[i] = "?"
|
||
args = append(args, code)
|
||
}
|
||
regionPlaceholder := strings.Join(placeholders, ",")
|
||
if db.isV3 {
|
||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||
COUNT(*) AS msg_count,
|
||
MAX(t.first_seen) AS last_activity,
|
||
(SELECT t2.decoded_json FROM transmissions t2
|
||
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
|
||
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
|
||
FROM transmissions t
|
||
JOIN observations o ON o.transmission_id = t.id
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.payload_type = 5
|
||
AND t.channel_hash IS NOT NULL
|
||
AND t.channel_hash NOT LIKE 'enc_%%'
|
||
AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)
|
||
GROUP BY t.channel_hash
|
||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||
} else {
|
||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||
COUNT(*) AS msg_count,
|
||
MAX(t.first_seen) AS last_activity,
|
||
(SELECT t2.decoded_json FROM transmissions t2
|
||
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
|
||
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
|
||
FROM transmissions t
|
||
JOIN observations o ON o.transmission_id = t.id
|
||
WHERE t.payload_type = 5
|
||
AND t.channel_hash IS NOT NULL
|
||
AND t.channel_hash NOT LIKE 'enc_%%'
|
||
AND EXISTS (
|
||
SELECT 1 FROM observers obs
|
||
WHERE obs.id = o.observer_id
|
||
AND UPPER(TRIM(obs.iata)) IN (%s)
|
||
)
|
||
GROUP BY t.channel_hash
|
||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||
}
|
||
} else {
|
||
querySQL = `SELECT channel_hash,
|
||
COUNT(*) AS msg_count,
|
||
MAX(first_seen) AS last_activity,
|
||
(SELECT t2.decoded_json FROM transmissions t2
|
||
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
|
||
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
|
||
FROM transmissions t
|
||
WHERE payload_type = 5
|
||
AND channel_hash IS NOT NULL
|
||
AND channel_hash NOT LIKE 'enc_%%'
|
||
GROUP BY channel_hash
|
||
ORDER BY last_activity DESC`
|
||
}
|
||
|
||
rows, err := db.conn.Query(querySQL, args...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
channels := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
var chHash, lastActivity, sampleJSON sql.NullString
|
||
var msgCount int
|
||
if err := rows.Scan(&chHash, &msgCount, &lastActivity, &sampleJSON); err != nil {
|
||
continue
|
||
}
|
||
channelName := nullStr(chHash)
|
||
if channelName == "" {
|
||
continue
|
||
}
|
||
|
||
var lastMessage, lastSender interface{}
|
||
if sampleJSON.Valid {
|
||
var decoded map[string]interface{}
|
||
if json.Unmarshal([]byte(sampleJSON.String), &decoded) == nil {
|
||
if text, ok := decoded["text"].(string); ok && text != "" {
|
||
idx := strings.Index(text, ": ")
|
||
if idx > 0 {
|
||
lastMessage = text[idx+2:]
|
||
} else {
|
||
lastMessage = text
|
||
}
|
||
if sender, ok := decoded["sender"].(string); ok {
|
||
lastSender = sender
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
channels = append(channels, map[string]interface{}{
|
||
"hash": channelName, "name": channelName,
|
||
"lastMessage": lastMessage, "lastSender": lastSender,
|
||
"messageCount": msgCount, "lastActivity": nullStr(lastActivity),
|
||
})
|
||
}
|
||
|
||
// Store in cache (60s TTL)
|
||
db.channelsCacheMu.Lock()
|
||
db.channelsCacheRes = channels
|
||
db.channelsCacheKey = regionParam
|
||
db.channelsCacheExp = time.Now().Add(60 * time.Second)
|
||
db.channelsCacheMu.Unlock()
|
||
|
||
return channels, nil
|
||
}
|
||
|
||
// GetEncryptedChannels returns channels where all messages are undecryptable (no key).
|
||
// Uses channel_hash column (prefixed with 'enc_') for fast grouped queries.
|
||
func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, error) {
|
||
regionParam := ""
|
||
if len(region) > 0 {
|
||
regionParam = region[0]
|
||
}
|
||
regionCodes := normalizeRegionCodes(regionParam)
|
||
|
||
var querySQL string
|
||
args := make([]interface{}, 0, len(regionCodes))
|
||
|
||
if len(regionCodes) > 0 {
|
||
placeholders := make([]string, len(regionCodes))
|
||
for i, code := range regionCodes {
|
||
placeholders[i] = "?"
|
||
args = append(args, code)
|
||
}
|
||
regionPlaceholder := strings.Join(placeholders, ",")
|
||
if db.isV3 {
|
||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||
COUNT(*) AS msg_count,
|
||
MAX(t.first_seen) AS last_activity
|
||
FROM transmissions t
|
||
JOIN observations o ON o.transmission_id = t.id
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.payload_type = 5
|
||
AND t.channel_hash LIKE 'enc_%%'
|
||
AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)
|
||
GROUP BY t.channel_hash
|
||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||
} else {
|
||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||
COUNT(*) AS msg_count,
|
||
MAX(t.first_seen) AS last_activity
|
||
FROM transmissions t
|
||
JOIN observations o ON o.transmission_id = t.id
|
||
WHERE t.payload_type = 5
|
||
AND t.channel_hash LIKE 'enc_%%'
|
||
AND EXISTS (
|
||
SELECT 1 FROM observers obs
|
||
WHERE obs.id = o.observer_id
|
||
AND UPPER(TRIM(obs.iata)) IN (%s)
|
||
)
|
||
GROUP BY t.channel_hash
|
||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||
}
|
||
} else {
|
||
querySQL = `SELECT channel_hash,
|
||
COUNT(*) AS msg_count,
|
||
MAX(first_seen) AS last_activity
|
||
FROM transmissions
|
||
WHERE payload_type = 5
|
||
AND channel_hash LIKE 'enc_%%'
|
||
GROUP BY channel_hash
|
||
ORDER BY last_activity DESC`
|
||
}
|
||
|
||
rows, err := db.conn.Query(querySQL, args...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
channels := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
var chHash, lastActivity sql.NullString
|
||
var msgCount int
|
||
if err := rows.Scan(&chHash, &msgCount, &lastActivity); err != nil {
|
||
continue
|
||
}
|
||
fullHash := nullStrVal(chHash) // e.g. "enc_3A"
|
||
hexPart := strings.TrimPrefix(fullHash, "enc_")
|
||
channels = append(channels, map[string]interface{}{
|
||
"hash": fullHash,
|
||
"name": "Encrypted (0x" + hexPart + ")",
|
||
"lastMessage": nil,
|
||
"lastSender": nil,
|
||
"messageCount": msgCount,
|
||
"lastActivity": nullStr(lastActivity),
|
||
"encrypted": true,
|
||
})
|
||
}
|
||
return channels, nil
|
||
}
|
||
|
||
// GetChannelMessages returns messages for a specific channel.
|
||
// Uses transmission-level ordering (first_seen) to ensure correct message
|
||
// sequence even when observations arrive out of order.
|
||
//
|
||
// Pagination is applied at the SQL level on the transmissions table (not on
|
||
// observations). The transmission.hash UNIQUE constraint means each
|
||
// transmission is one logical message; multiple observations of the same
|
||
// transmission collapse into one row with `repeats` = observation count.
|
||
// This avoids loading every observation row for a channel into Go memory
|
||
// before paginating (issue #1225: 5703 tx × ~50 obs ≈ 275K rows → ~30s
|
||
// for limit=50).
|
||
// channelMentionPrefixRe strips a leading "@target " reply-address the
|
||
// same way the frontend does (public/channels.js replyMatch) before
|
||
// matching the ping trigger, so "@CoreScopeBot ping" triggers the same as
|
||
// a bare "ping".
|
||
var channelMentionPrefixRe = regexp.MustCompile(`^@[A-Za-z0-9_-]{1,32}\s+`)
|
||
|
||
// pingTriggerWords are the exact (case-insensitive) message bodies that
|
||
// trigger a pong reply. Mirrored by pingTriggerWords in
|
||
// public/channels.js -- keep both lists in sync by hand.
|
||
var pingTriggerWords = map[string]bool{
|
||
"ping": true,
|
||
"/ping": true,
|
||
}
|
||
|
||
// isPingTrigger reports whether displayText, after stripping a leading
|
||
// "@target " mention the same way the frontend does (public/channels.js
|
||
// replyMatch), exactly matches one of pingTriggerWords.
|
||
func isPingTrigger(displayText string) bool {
|
||
trigger := strings.TrimSpace(displayText)
|
||
trigger = channelMentionPrefixRe.ReplaceAllString(trigger, "")
|
||
return pingTriggerWords[strings.ToLower(strings.TrimSpace(trigger))]
|
||
}
|
||
|
||
// pingBotReply synthesizes a "pong" reply for a channel message whose
|
||
// text matched isPingTrigger — CoreScope-side only, never transmitted
|
||
// back onto the mesh (CoreScope has no publish path to a MeshCore
|
||
// broker/radio). Purely a read-time annotation over data this message's
|
||
// own row already carries (hop count + relay path, SNR, hearing
|
||
// observer, region scope), not a persisted message.
|
||
//
|
||
// repeaterNames is the resolved relay path in hop order (element i is
|
||
// hop i's node name, falling back to its pubkey/hash-prefix when a name
|
||
// couldn't be resolved); nil/empty when hops == 0 or resolution wasn't
|
||
// available -- the hop count itself is unaffected either way.
|
||
//
|
||
// farthestKm is the same "how wide did this spread geographically" signal
|
||
// View Path's map shows (farthest any hearing station was from whoever
|
||
// heard it first), computed from stations' own GPS fixes only -- nil when
|
||
// fewer than two distinct stations have a position on file.
|
||
func pingBotReply(hops int, snr sql.NullFloat64, observer string, repeaterNames []string, farthestKm *float64) map[string]interface{} {
|
||
parts := make([]string, 0, 4)
|
||
if hops > 0 {
|
||
s := "s"
|
||
if hops == 1 {
|
||
s = ""
|
||
}
|
||
hopDesc := fmt.Sprintf("%d hop%s", hops, s)
|
||
if len(repeaterNames) > 0 {
|
||
hopDesc += " (via " + strings.Join(repeaterNames, " → ") + ")"
|
||
}
|
||
parts = append(parts, hopDesc)
|
||
} else {
|
||
parts = append(parts, "0 hops (direct)")
|
||
}
|
||
if snr.Valid {
|
||
parts = append(parts, fmt.Sprintf("SNR %.1fdB", snr.Float64))
|
||
}
|
||
if farthestKm != nil {
|
||
parts = append(parts, fmt.Sprintf("spread up to %.1fkm", *farthestKm))
|
||
}
|
||
if observer != "" {
|
||
parts = append(parts, "heard by "+observer)
|
||
}
|
||
// Deliberately no scope/area here -- both are already shown on the
|
||
// triggering message's own meta line right above this reply, so
|
||
// repeating them would just be redundant.
|
||
return map[string]interface{}{
|
||
"sender": "CoreScopeBot",
|
||
"text": "🏓 pong! " + strings.Join(parts, " · "),
|
||
"hops": hops,
|
||
"snr": nullFloat(snr),
|
||
}
|
||
}
|
||
|
||
func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region ...string) ([]map[string]interface{}, int, error) {
|
||
if limit <= 0 {
|
||
limit = 100
|
||
}
|
||
if offset < 0 {
|
||
offset = 0
|
||
}
|
||
|
||
regionParam := ""
|
||
if len(region) > 0 {
|
||
regionParam = region[0]
|
||
}
|
||
regionCodes := normalizeRegionCodes(regionParam)
|
||
regionArgs := make([]interface{}, 0, len(regionCodes))
|
||
regionPlaceholders := ""
|
||
if len(regionCodes) > 0 {
|
||
placeholders := make([]string, len(regionCodes))
|
||
for i, code := range regionCodes {
|
||
placeholders[i] = "?"
|
||
regionArgs = append(regionArgs, code)
|
||
}
|
||
regionPlaceholders = strings.Join(placeholders, ",")
|
||
}
|
||
|
||
// regionFilter: a transmission is included only if at least one of its
|
||
// observations has an observer in one of the requested regions.
|
||
regionFilter := ""
|
||
if len(regionCodes) > 0 {
|
||
if db.isV3 {
|
||
regionFilter = fmt.Sprintf(` AND EXISTS (
|
||
SELECT 1 FROM observations o
|
||
JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE o.transmission_id = t.id
|
||
AND UPPER(TRIM(obs.iata)) IN (%s))`, regionPlaceholders)
|
||
} else {
|
||
regionFilter = fmt.Sprintf(` AND EXISTS (
|
||
SELECT 1 FROM observations o
|
||
JOIN observers obs ON obs.id = o.observer_id
|
||
WHERE o.transmission_id = t.id
|
||
AND UPPER(TRIM(obs.iata)) IN (%s))`, regionPlaceholders)
|
||
}
|
||
}
|
||
|
||
// 1) Total count (after region filter, before pagination).
|
||
countSQL := `SELECT COUNT(*) FROM transmissions t
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter
|
||
countArgs := []interface{}{channelHash}
|
||
countArgs = append(countArgs, regionArgs...)
|
||
var total int
|
||
if err := db.conn.QueryRow(countSQL, countArgs...).Scan(&total); err != nil {
|
||
return nil, 0, err
|
||
}
|
||
|
||
// 2) Page of transmission IDs — newest LIMIT msgs minus OFFSET.
|
||
// Issue #1366 follow-up (fix #2): select page by latest observation
|
||
// timestamp (LatestSeen) DESC, NOT by t.first_seen DESC — otherwise
|
||
// a heartbeat tx whose FirstSeen is 24h old but whose latest
|
||
// observation is fresh gets pushed off page 1.
|
||
//
|
||
// PR #1368 perf fix: use a correlated subquery for MAX(timestamp) per
|
||
// transmission. With the composite index idx_observations_tx_ts
|
||
// (transmission_id, timestamp) sqlite resolves MAX as an index-only
|
||
// rightmost-leaf lookup — total O(N_tx · log N_obs). The previously-
|
||
// used grouped derived table (`GROUP BY transmission_id` over the
|
||
// whole observations table) scanned all observation rows (O(N_obs))
|
||
// and blew the 1.5s perf budget on 1500 tx × 50 obs under -race.
|
||
// LEFT JOIN + GROUP BY t.id was even slower because GROUP BY forced
|
||
// a temp B-tree on the full transmissions×observations join.
|
||
//
|
||
// The returned page is in newest-LatestSeen-FIRST (DESC) order.
|
||
// The Go side re-orders the emitted rows ASC below (fix #3) so the
|
||
// contract matches the in-memory path's tail-of-msgOrder convention.
|
||
pageSQL := `SELECT t.id,
|
||
COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch
|
||
FROM transmissions t
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5
|
||
ORDER BY latest_obs_epoch DESC, t.id DESC
|
||
LIMIT ? OFFSET ?`
|
||
if len(regionCodes) > 0 {
|
||
pageSQL = `SELECT t.id,
|
||
COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch
|
||
FROM transmissions t
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + `
|
||
ORDER BY latest_obs_epoch DESC, t.id DESC
|
||
LIMIT ? OFFSET ?`
|
||
}
|
||
pageArgs := []interface{}{channelHash}
|
||
pageArgs = append(pageArgs, regionArgs...)
|
||
pageArgs = append(pageArgs, limit, offset)
|
||
|
||
idRows, err := db.conn.Query(pageSQL, pageArgs...)
|
||
if err != nil {
|
||
return nil, 0, err
|
||
}
|
||
pageIDs := make([]int, 0, limit)
|
||
for idRows.Next() {
|
||
var id int
|
||
var le sql.NullInt64
|
||
if err := idRows.Scan(&id, &le); err == nil {
|
||
pageIDs = append(pageIDs, id)
|
||
}
|
||
}
|
||
idRows.Close()
|
||
|
||
if len(pageIDs) == 0 {
|
||
return []map[string]interface{}{}, total, nil
|
||
}
|
||
|
||
// 3) Fetch observations for just this page of transmissions. We keep
|
||
// the original "first observation wins" semantic for hops/snr/observer
|
||
// by ordering observations by id ASC and breaking after first per tx.
|
||
idPlaceholders := make([]string, len(pageIDs))
|
||
obsArgs := make([]interface{}, len(pageIDs))
|
||
for i, id := range pageIDs {
|
||
idPlaceholders[i] = "?"
|
||
obsArgs[i] = id
|
||
}
|
||
scopeCol := ""
|
||
if db.hasScopeName {
|
||
scopeCol = ", t.scope_name"
|
||
}
|
||
// resolvedPathCol feeds the ping-bot reply's "via RepeaterA → RepeaterB"
|
||
// hop names (see the bulk-resolve pass below) -- optional like
|
||
// scopeCol since not every DB/test fixture has this column.
|
||
resolvedPathCol := ""
|
||
if db.hasResolvedPath {
|
||
resolvedPathCol = ", o.resolved_path"
|
||
}
|
||
var obsSQL string
|
||
if db.isV3 {
|
||
obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen,
|
||
obs.id, obs.name, o.snr, o.path_json, o.timestamp, t.route_type` + scopeCol + resolvedPathCol + `
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.id IN (` + strings.Join(idPlaceholders, ",") + `)
|
||
ORDER BY o.id ASC`
|
||
} else {
|
||
obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen,
|
||
o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp, t.route_type` + scopeCol + resolvedPathCol + `
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
WHERE t.id IN (` + strings.Join(idPlaceholders, ",") + `)
|
||
ORDER BY o.id ASC`
|
||
}
|
||
|
||
rows, err := db.conn.Query(obsSQL, obsArgs...)
|
||
if err != nil {
|
||
return nil, 0, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
type msg struct {
|
||
Data map[string]interface{}
|
||
Repeats int
|
||
LatestEpoch int64 // max observation timestamp (unix seconds) — issue #1366
|
||
}
|
||
msgMap := make(map[int]*msg, len(pageIDs))
|
||
|
||
// pendingPing collects a ping-triggering message's REACH across every
|
||
// observation of it, not just the first: hops/snr/resolvedPath track
|
||
// the DEEPEST (max-hop) observation seen so far -- how far the packet
|
||
// had propagated before the farthest-along observer heard it -- and
|
||
// observers is every distinct observer that heard it at all (breadth).
|
||
// A single arbitrary "first observation wins" data point understates
|
||
// both: two observers can hear the same flood at very different hop
|
||
// depths depending on which relay leg reached them.
|
||
type pendingPing struct {
|
||
hops int
|
||
snr sql.NullFloat64
|
||
resolvedPath []*string
|
||
observers map[string]bool
|
||
// observerPubkeys/firstPubkey/firstTS feed the "spread up to Nkm"
|
||
// reply text -- the same farthest-from-first-hearer distance View
|
||
// Path shows on its map, computed here as a cheap position-only
|
||
// pass (no neighbor-centroid approximation) rather than reusing
|
||
// GetPacketPath's heavier per-branch query for every ping.
|
||
observerPubkeys map[string]bool
|
||
firstPubkey string
|
||
firstTS int64
|
||
}
|
||
pendingPings := make(map[int]*pendingPing)
|
||
|
||
for rows.Next() {
|
||
var pktID, txID int
|
||
var pktHash, dj, fs, obsID, obsName, pathJSON, resolvedPathJSON sql.NullString
|
||
var snr sql.NullFloat64
|
||
var obsTs sql.NullInt64
|
||
var routeType sql.NullInt64
|
||
var scopeName sql.NullString
|
||
scanArgs := []interface{}{&pktID, &txID, &pktHash, &dj, &fs, &obsID, &obsName, &snr, &pathJSON, &obsTs, &routeType}
|
||
if db.hasScopeName {
|
||
scanArgs = append(scanArgs, &scopeName)
|
||
}
|
||
if db.hasResolvedPath {
|
||
scanArgs = append(scanArgs, &resolvedPathJSON)
|
||
}
|
||
if err := rows.Scan(scanArgs...); err != nil {
|
||
return nil, 0, err
|
||
}
|
||
if !dj.Valid {
|
||
continue
|
||
}
|
||
|
||
// Hop count, relay path, and hearing station for THIS observation
|
||
// row -- computed for every row (not just the first) so a ping's
|
||
// reach can be tracked across every station that heard it.
|
||
var hops int
|
||
var entryPrefix string
|
||
if pathJSON.Valid {
|
||
var h []string
|
||
if json.Unmarshal([]byte(pathJSON.String), &h) == nil {
|
||
hops = len(h)
|
||
if len(h) > 0 {
|
||
entryPrefix = h[0]
|
||
}
|
||
}
|
||
}
|
||
var resolvedPath []*string
|
||
if resolvedPathJSON.Valid {
|
||
resolvedPath = unmarshalResolvedPath(resolvedPathJSON.String)
|
||
}
|
||
observerName := ""
|
||
if obsName.Valid {
|
||
observerName = obsName.String
|
||
} else if obsID.Valid {
|
||
observerName = obsID.String
|
||
}
|
||
|
||
if existing, ok := msgMap[txID]; ok {
|
||
existing.Repeats++
|
||
if obsTs.Valid && obsTs.Int64 > existing.LatestEpoch {
|
||
existing.LatestEpoch = obsTs.Int64
|
||
}
|
||
if agg, ok := pendingPings[txID]; ok {
|
||
if observerName != "" {
|
||
agg.observers[observerName] = true
|
||
}
|
||
if obsID.Valid && obsID.String != "" {
|
||
pk := strings.ToLower(strings.TrimSpace(obsID.String))
|
||
agg.observerPubkeys[pk] = true
|
||
if obsTs.Valid && (agg.firstPubkey == "" || obsTs.Int64 < agg.firstTS) {
|
||
agg.firstPubkey = pk
|
||
agg.firstTS = obsTs.Int64
|
||
}
|
||
}
|
||
if hops > agg.hops {
|
||
agg.hops, agg.snr, agg.resolvedPath = hops, snr, resolvedPath
|
||
}
|
||
}
|
||
continue
|
||
}
|
||
var decoded map[string]interface{}
|
||
if json.Unmarshal([]byte(dj.String), &decoded) != nil {
|
||
continue
|
||
}
|
||
text, _ := decoded["text"].(string)
|
||
sender, _ := decoded["sender"].(string)
|
||
if sender == "" && text != "" {
|
||
if idx := strings.Index(text, ": "); idx > 0 && idx < 50 {
|
||
sender = text[:idx]
|
||
}
|
||
}
|
||
displaySender := sender
|
||
displayText := text
|
||
if text != "" {
|
||
if idx := strings.Index(text, ": "); idx > 0 && idx < 50 {
|
||
displaySender = text[:idx]
|
||
displayText = text[idx+2:]
|
||
}
|
||
}
|
||
senderTs := decoded["sender_timestamp"]
|
||
// entryObserverPubkey: a 0-hop (direct) reception has no relay path,
|
||
// so entryPrefix is empty and annotateMessageAreas has nothing to
|
||
// resolve -- even though the hearing station's own position is a
|
||
// reasonable stand-in for "where this happened" at 0 hops. Only
|
||
// captured for the direct case; a message with an unresolvable
|
||
// multi-hop path deliberately still gets no fallback (an estimate
|
||
// from the wrong end of a relay chain isn't worth showing).
|
||
var entryObserverPubkey string
|
||
if hops == 0 && obsID.Valid && obsID.String != "" {
|
||
entryObserverPubkey = strings.ToLower(strings.TrimSpace(obsID.String))
|
||
}
|
||
m := &msg{
|
||
Data: map[string]interface{}{
|
||
"sender": displaySender,
|
||
"text": displayText,
|
||
"timestamp": nullStr(fs),
|
||
"first_seen": nullStr(fs),
|
||
"sender_timestamp": senderTs,
|
||
"packetId": pktID,
|
||
"packetHash": nullStr(pktHash),
|
||
"repeats": 1,
|
||
"observers": []string{},
|
||
"hops": hops,
|
||
"snr": nullFloat(snr),
|
||
"scope": nullStr(scopeName),
|
||
"routeType": nullInt(routeType),
|
||
"entryPrefix": entryPrefix,
|
||
"entryObserverPubkey": entryObserverPubkey,
|
||
},
|
||
Repeats: 1,
|
||
}
|
||
if obsTs.Valid {
|
||
m.LatestEpoch = obsTs.Int64
|
||
}
|
||
if observerName != "" {
|
||
m.Data["observers"] = []string{observerName}
|
||
}
|
||
if isPingTrigger(displayText) {
|
||
agg := &pendingPing{
|
||
hops: hops, snr: snr, resolvedPath: resolvedPath,
|
||
observers: map[string]bool{}, observerPubkeys: map[string]bool{},
|
||
}
|
||
if observerName != "" {
|
||
agg.observers[observerName] = true
|
||
}
|
||
if obsID.Valid && obsID.String != "" {
|
||
pk := strings.ToLower(strings.TrimSpace(obsID.String))
|
||
agg.observerPubkeys[pk] = true
|
||
if obsTs.Valid {
|
||
agg.firstPubkey = pk
|
||
agg.firstTS = obsTs.Int64
|
||
}
|
||
}
|
||
pendingPings[txID] = agg
|
||
}
|
||
msgMap[txID] = m
|
||
}
|
||
|
||
// Bulk-resolve every pubkey referenced by any ping's DEEPEST relay path
|
||
// in ONE query, then build each pending reply's "via RepeaterA →
|
||
// RepeaterB" text plus its observer-breadth label. Names default to
|
||
// the raw pubkey/prefix when unresolved rather than being dropped, so
|
||
// the hop count and reply still make sense.
|
||
if len(pendingPings) > 0 {
|
||
pubkeySet := map[string]bool{}
|
||
for _, p := range pendingPings {
|
||
for _, pk := range p.resolvedPath {
|
||
if pk != nil && *pk != "" {
|
||
pubkeySet[*pk] = true
|
||
}
|
||
}
|
||
}
|
||
pubkeys := make([]string, 0, len(pubkeySet))
|
||
for pk := range pubkeySet {
|
||
pubkeys = append(pubkeys, pk)
|
||
}
|
||
names, _ := db.namesAndRolesForPubkeys(pubkeys)
|
||
|
||
// Bulk-resolve observer positions too, for the "spread up to Nkm"
|
||
// part of the reply -- only for pings that could possibly show one
|
||
// (a first-hearer plus at least one other distinct station), and
|
||
// deliberately WITHOUT GetPacketPath's neighbor-centroid fallback
|
||
// for unpositioned stations: that's a per-node query each, too
|
||
// expensive to run for every ping on a page of channel messages.
|
||
// A station missing its own GPS fix just doesn't contribute here.
|
||
posPubkeySet := map[string]bool{}
|
||
for _, p := range pendingPings {
|
||
if p.firstPubkey == "" || len(p.observerPubkeys) < 2 {
|
||
continue
|
||
}
|
||
for pk := range p.observerPubkeys {
|
||
posPubkeySet[pk] = true
|
||
}
|
||
}
|
||
posByPK := map[string][2]float64{}
|
||
if len(posPubkeySet) > 0 {
|
||
posPubkeys := make([]string, 0, len(posPubkeySet))
|
||
for pk := range posPubkeySet {
|
||
posPubkeys = append(posPubkeys, pk)
|
||
}
|
||
placeholders := make([]byte, 0, len(posPubkeys)*2)
|
||
args := make([]interface{}, len(posPubkeys))
|
||
for i, pk := range posPubkeys {
|
||
if i > 0 {
|
||
placeholders = append(placeholders, ',')
|
||
}
|
||
placeholders = append(placeholders, '?')
|
||
args[i] = pk
|
||
}
|
||
posRows, err := db.conn.Query(
|
||
"SELECT public_key, lat, lon FROM nodes WHERE public_key IN ("+string(placeholders)+")", args...)
|
||
if err == nil {
|
||
for posRows.Next() {
|
||
var pk string
|
||
var lat, lon sql.NullFloat64
|
||
// (0,0) is the ocean off Ghana, not a real fix -- same
|
||
// exclusion GetPacketPath applies.
|
||
if posRows.Scan(&pk, &lat, &lon) == nil && lat.Valid && lon.Valid && !(lat.Float64 == 0 && lon.Float64 == 0) {
|
||
posByPK[pk] = [2]float64{lat.Float64, lon.Float64}
|
||
}
|
||
}
|
||
posRows.Close()
|
||
}
|
||
}
|
||
|
||
for txID, p := range pendingPings {
|
||
var repeaterNames []string
|
||
for _, pk := range p.resolvedPath {
|
||
if pk == nil || *pk == "" {
|
||
continue
|
||
}
|
||
if name := names[*pk]; name != "" {
|
||
repeaterNames = append(repeaterNames, name)
|
||
} else {
|
||
repeaterNames = append(repeaterNames, *pk)
|
||
}
|
||
}
|
||
// Breadth: name the single observer when there's only one (as
|
||
// specific as before), otherwise report the count -- "heard by
|
||
// 4 observers" says more about actual reach than an arbitrarily
|
||
// picked single name once more than one observer heard it.
|
||
observerLabel := ""
|
||
switch len(p.observers) {
|
||
case 0:
|
||
// leave empty
|
||
case 1:
|
||
for name := range p.observers {
|
||
observerLabel = name
|
||
}
|
||
default:
|
||
observerLabel = fmt.Sprintf("%d observers", len(p.observers))
|
||
}
|
||
var farthestKm *float64
|
||
if firstPos, ok := posByPK[p.firstPubkey]; ok {
|
||
var maxKm float64
|
||
sawOther := false
|
||
for pk := range p.observerPubkeys {
|
||
if pk == p.firstPubkey {
|
||
continue
|
||
}
|
||
if pos, ok2 := posByPK[pk]; ok2 {
|
||
d := haversineKm(firstPos[0], firstPos[1], pos[0], pos[1])
|
||
if !sawOther || d > maxKm {
|
||
maxKm, sawOther = d, true
|
||
}
|
||
}
|
||
}
|
||
if sawOther {
|
||
farthestKm = &maxKm
|
||
}
|
||
}
|
||
if m, ok := msgMap[txID]; ok {
|
||
br := pingBotReply(p.hops, p.snr, observerLabel, repeaterNames, farthestKm)
|
||
// touchedObserverPubkeys never reaches the client -- routes.go's
|
||
// annotateBotReplyTouchedAreas resolves it to area labels
|
||
// (needs s.cfg.Areas, not available at this SQL-only DB
|
||
// layer) and deletes it before the response is written.
|
||
if len(p.observerPubkeys) > 0 {
|
||
pks := make([]string, 0, len(p.observerPubkeys))
|
||
for pk := range p.observerPubkeys {
|
||
pks = append(pks, pk)
|
||
}
|
||
br["touchedObserverPubkeys"] = pks
|
||
}
|
||
m.Data["botReply"] = br
|
||
}
|
||
}
|
||
}
|
||
|
||
// Issue #1366 follow-up: emit batch sorted by LatestSeen ascending
|
||
// (newest LAST) — matches the in-memory path's tail-of-msgOrder
|
||
// convention and the frontend's scrollToBottom() behavior. pageIDs
|
||
// order is not LatestSeen-ordered for in-page rows after fix #2.
|
||
type emitted struct {
|
||
latestEpoch int64
|
||
txID int
|
||
data map[string]interface{}
|
||
}
|
||
rowsOut := make([]emitted, 0, len(pageIDs))
|
||
for _, id := range pageIDs {
|
||
m, ok := msgMap[id]
|
||
if !ok {
|
||
// Transmission had no observations (shouldn't happen via normal
|
||
// ingest) or decoded_json was NULL/invalid — skip silently to
|
||
// preserve prior behavior.
|
||
continue
|
||
}
|
||
m.Data["repeats"] = m.Repeats
|
||
// Issue #1366: emit LatestSeen (max obs timestamp) as the rendered
|
||
// `timestamp` field. `first_seen` stays alongside for debug.
|
||
if m.LatestEpoch > 0 {
|
||
m.Data["timestamp"] = time.Unix(m.LatestEpoch, 0).UTC().Format(time.RFC3339)
|
||
}
|
||
rowsOut = append(rowsOut, emitted{latestEpoch: m.LatestEpoch, txID: id, data: m.Data})
|
||
}
|
||
sort.SliceStable(rowsOut, func(i, j int) bool {
|
||
if rowsOut[i].latestEpoch != rowsOut[j].latestEpoch {
|
||
return rowsOut[i].latestEpoch < rowsOut[j].latestEpoch
|
||
}
|
||
return rowsOut[i].txID < rowsOut[j].txID
|
||
})
|
||
messages := make([]map[string]interface{}, 0, len(rowsOut))
|
||
for _, e := range rowsOut {
|
||
messages = append(messages, e.data)
|
||
}
|
||
|
||
return messages, total, nil
|
||
}
|
||
|
||
// GetNewTransmissionsSince returns new transmissions after a given ID for WebSocket polling.
|
||
func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]interface{}, error) {
|
||
if limit <= 0 {
|
||
limit = 100
|
||
}
|
||
rows, err := db.conn.Query(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json
|
||
FROM transmissions t WHERE t.id > ? ORDER BY t.id ASC LIMIT ?`, lastID, limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
var result []map[string]interface{}
|
||
for rows.Next() {
|
||
var id int
|
||
var rawHex, hash, firstSeen, decodedJSON sql.NullString
|
||
var routeType, payloadType, payloadVersion sql.NullInt64
|
||
rows.Scan(&id, &rawHex, &hash, &firstSeen, &routeType, &payloadType, &payloadVersion, &decodedJSON)
|
||
result = append(result, map[string]interface{}{
|
||
"id": id,
|
||
"raw_hex": nullStr(rawHex),
|
||
"hash": nullStr(hash),
|
||
"first_seen": nullStr(firstSeen),
|
||
"route_type": nullInt(routeType),
|
||
"payload_type": nullInt(payloadType),
|
||
"payload_version": nullInt(payloadVersion),
|
||
"decoded_json": nullStr(decodedJSON),
|
||
})
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// GetMaxTransmissionID returns the current max ID for polling.
|
||
func (db *DB) GetMaxTransmissionID() int {
|
||
var maxID int
|
||
db.conn.QueryRow("SELECT COALESCE(MAX(id), 0) FROM transmissions").Scan(&maxID)
|
||
return maxID
|
||
}
|
||
|
||
// GetMaxObservationID returns the current max observation ID for polling.
|
||
func (db *DB) GetMaxObservationID() int {
|
||
var maxID int
|
||
db.conn.QueryRow("SELECT COALESCE(MAX(id), 0) FROM observations").Scan(&maxID)
|
||
return maxID
|
||
}
|
||
|
||
// GetObserverPacketCounts returns packetsLastHour for all observers (batch query).
|
||
func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int {
|
||
counts := make(map[string]int)
|
||
var rows *sql.Rows
|
||
var err error
|
||
if db.isV3 {
|
||
rows, err = db.conn.Query(`SELECT obs.id, COUNT(*) as cnt
|
||
FROM observations o
|
||
JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE o.timestamp > ?
|
||
GROUP BY obs.id`, sinceEpoch)
|
||
} else {
|
||
rows, err = db.conn.Query(`SELECT o.observer_id, COUNT(*) as cnt
|
||
FROM observations o
|
||
WHERE o.observer_id IS NOT NULL AND o.timestamp > ?
|
||
GROUP BY o.observer_id`, sinceEpoch)
|
||
}
|
||
if err != nil {
|
||
return counts
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var id string
|
||
var cnt int
|
||
rows.Scan(&id, &cnt)
|
||
counts[id] = cnt
|
||
}
|
||
return counts
|
||
}
|
||
|
||
// GetNodeLocations returns a map of lowercase public_key → {lat, lon, role} for node geo lookups.
|
||
func (db *DB) GetNodeLocations() map[string]map[string]interface{} {
|
||
result := make(map[string]map[string]interface{})
|
||
rows, err := db.conn.Query("SELECT public_key, lat, lon, role FROM nodes")
|
||
if err != nil {
|
||
return result
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var pk string
|
||
var role sql.NullString
|
||
var lat, lon sql.NullFloat64
|
||
rows.Scan(&pk, &lat, &lon, &role)
|
||
result[strings.ToLower(pk)] = map[string]interface{}{
|
||
"lat": nullFloat(lat),
|
||
"lon": nullFloat(lon),
|
||
"role": nullStr(role),
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// GetNodeLocationsByKeys returns location data only for the given public keys.
|
||
// This avoids fetching ALL nodes when only a few keys need to be matched.
|
||
func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interface{} {
|
||
result := make(map[string]map[string]interface{})
|
||
if len(keys) == 0 {
|
||
return result
|
||
}
|
||
placeholders := make([]string, len(keys))
|
||
args := make([]interface{}, len(keys))
|
||
for i, k := range keys {
|
||
placeholders[i] = "?"
|
||
args[i] = strings.ToLower(k)
|
||
}
|
||
// #1481 P0-3: drop LOWER(public_key) — that wrap is non-sargable and
|
||
// forces a full scan. Nodes are stored lowercase already; we lowercase
|
||
// args in Go above so a plain IN matches the index on public_key.
|
||
query := "SELECT public_key, lat, lon, role FROM nodes WHERE public_key IN (" + strings.Join(placeholders, ",") + ")"
|
||
rows, err := db.conn.Query(query, args...)
|
||
if err != nil {
|
||
return result
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var pk string
|
||
var role sql.NullString
|
||
var lat, lon sql.NullFloat64
|
||
rows.Scan(&pk, &lat, &lon, &role)
|
||
result[strings.ToLower(pk)] = map[string]interface{}{
|
||
"lat": nullFloat(lat),
|
||
"lon": nullFloat(lon),
|
||
"role": nullStr(role),
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// GetRepeaterNamesByKeys batch-resolves pubkey -> display name, restricted
|
||
// to role IN ('repeater','room'). The candidate key set (e.g. from
|
||
// PacketStore.byPathHop) mixes full pubkeys with short hex-prefix bucket
|
||
// keys used internally for ambiguous-hop resolution (#1751 follow-up) —
|
||
// those never match a real nodes.public_key row, so the role-filtered IN
|
||
// query doubles as the "is this actually a distinct node" existence check.
|
||
// A matched pubkey with an empty/unset name falls back to itself so a
|
||
// real repeater is never silently dropped just because it has no name yet.
|
||
//
|
||
// Queried in chunks of repeaterNamesByKeysBatchSize — SQLite's default
|
||
// SQLITE_MAX_VARIABLE_NUMBER is 999 on older builds, and a large mesh's
|
||
// byPathHop candidate set can exceed that in one IN (...) clause.
|
||
const repeaterNamesByKeysBatchSize = 500
|
||
|
||
func (db *DB) GetRepeaterNamesByKeys(keys []string) map[string]string {
|
||
result := make(map[string]string)
|
||
if len(keys) == 0 {
|
||
return result
|
||
}
|
||
for start := 0; start < len(keys); start += repeaterNamesByKeysBatchSize {
|
||
end := start + repeaterNamesByKeysBatchSize
|
||
if end > len(keys) {
|
||
end = len(keys)
|
||
}
|
||
chunk := keys[start:end]
|
||
placeholders := make([]string, len(chunk))
|
||
args := make([]interface{}, len(chunk))
|
||
for i, k := range chunk {
|
||
placeholders[i] = "?"
|
||
args[i] = strings.ToLower(k)
|
||
}
|
||
query := "SELECT public_key, name FROM nodes WHERE role IN ('repeater','room') AND public_key IN (" + strings.Join(placeholders, ",") + ")"
|
||
rows, err := db.conn.Query(query, args...)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
for rows.Next() {
|
||
var pk string
|
||
var name sql.NullString
|
||
if rows.Scan(&pk, &name) != nil {
|
||
continue
|
||
}
|
||
pk = strings.ToLower(pk)
|
||
if name.Valid && name.String != "" {
|
||
result[pk] = name.String
|
||
} else {
|
||
result[pk] = pk
|
||
}
|
||
}
|
||
rows.Close()
|
||
}
|
||
return result
|
||
}
|
||
|
||
// GetNodesByDefaultScope groups nodes by their own configured region
|
||
// (nodes.default_scope, #899) — the region a node's ADVERTs actually carry,
|
||
// as opposed to GetRepeaterNamesByKeys/TransportedScopes which is about
|
||
// repeaters *relaying* traffic scoped by other senders. Distinguishes
|
||
// "runs this region" from "has carried this region's traffic".
|
||
func (db *DB) GetNodesByDefaultScope() (map[string][]RepeaterRef, error) {
|
||
result := make(map[string][]RepeaterRef)
|
||
if !db.hasDefaultScope {
|
||
return result, nil
|
||
}
|
||
rows, err := db.conn.Query(`SELECT public_key, name, default_scope FROM nodes WHERE default_scope IS NOT NULL AND default_scope != ''`)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("nodes by default_scope query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var pk, scope string
|
||
var name sql.NullString
|
||
if rows.Scan(&pk, &name, &scope) != nil {
|
||
continue
|
||
}
|
||
displayName := pk
|
||
if name.Valid && name.String != "" {
|
||
displayName = name.String
|
||
}
|
||
result[scope] = append(result[scope], RepeaterRef{Name: displayName, PublicKey: strings.ToLower(pk)})
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// nodeAreaScopeInput is one node's position + name + default_scope +
|
||
// pubkey — the raw input to computeScopeAdoptionByArea. DefaultScope is ""
|
||
// when unset or when this DB predates #899 (no default_scope column at
|
||
// all). PublicKey is lowercase, for looking a node up in a
|
||
// RepeaterRelayInfo map.
|
||
type nodeAreaScopeInput struct {
|
||
PublicKey string
|
||
Name string
|
||
Lat, Lon float64
|
||
DefaultScope string
|
||
}
|
||
|
||
// GetNodesForScopeAdoption returns every node with a real GPS fix (0,0
|
||
// excluded, same convention as geofilter.PassesFilter) and its
|
||
// default_scope, for computeScopeAdoptionByArea to bucket by configured
|
||
// area. Unlike GetNodesByDefaultScope, this includes nodes with NO scope
|
||
// too — the whole point is measuring adoption, not just listing who has one.
|
||
func (db *DB) GetNodesForScopeAdoption() ([]nodeAreaScopeInput, error) {
|
||
query := "SELECT public_key, name, lat, lon"
|
||
if db.hasDefaultScope {
|
||
query += ", default_scope"
|
||
}
|
||
query += " FROM nodes WHERE lat IS NOT NULL AND lon IS NOT NULL AND lat != 0 AND lon != 0"
|
||
rows, err := db.conn.Query(query)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("nodes for scope adoption query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
var out []nodeAreaScopeInput
|
||
for rows.Next() {
|
||
var pk string
|
||
var name sql.NullString
|
||
var lat, lon float64
|
||
var scope sql.NullString
|
||
var scanErr error
|
||
if db.hasDefaultScope {
|
||
scanErr = rows.Scan(&pk, &name, &lat, &lon, &scope)
|
||
} else {
|
||
scanErr = rows.Scan(&pk, &name, &lat, &lon)
|
||
}
|
||
if scanErr != nil {
|
||
continue
|
||
}
|
||
displayName := pk
|
||
if name.Valid && name.String != "" {
|
||
displayName = name.String
|
||
}
|
||
out = append(out, nodeAreaScopeInput{PublicKey: strings.ToLower(pk), Name: displayName, Lat: lat, Lon: lon, DefaultScope: scope.String})
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// computeScopeAdoptionByArea buckets nodes by their most specific
|
||
// configured area and tallies, per area: how many nodes sit there at all,
|
||
// how many "use scope" in ANY sense, and (when the area itself has a
|
||
// RegionScopes link) how many specifically use THAT region — i.e. does
|
||
// this geographic community actually engage with the scope the area is
|
||
// nominally tied to, or something else entirely (or nothing at all). A
|
||
// node outside every configured area is excluded.
|
||
//
|
||
// Unlike the per-node area *badges* (AreaForPoint/AreaKeyForPoint, which
|
||
// pick a single most-specific area), this uses AreaKeysForPoint so a node
|
||
// counts toward EVERY containing area — a node in "Aarhus by" also counts
|
||
// toward "Jylland" and "Danmark (alle)". Without this, a broad roll-up
|
||
// area like "Danmark (alle)" would only ever show the handful of nodes not
|
||
// claimed by any smaller, more specific area (dborup flagged this: DK
|
||
// showed almost nothing because nearly every real node already belonged
|
||
// to a narrower sub-area), instead of the whole country's actual adoption.
|
||
//
|
||
// "Uses scope" counts two distinct signals, same runs-this-region vs
|
||
// carried-this-region's-traffic distinction as OriginatingNodesByRegion vs
|
||
// RepeatersByRegion above: (1) the node's own default_scope, and (2) any
|
||
// region it has ever RELAYED (relayInfo/TransportedScopes) — a repeater
|
||
// can carry dk-horsens traffic and thereby support the Horsens area
|
||
// without ever configuring dk-horsens as its own default_scope. relayInfo
|
||
// may be nil (in-memory store unavailable), in which case matching falls
|
||
// back to default_scope only.
|
||
//
|
||
// For areas with a RegionScopes link, also returns the actual node lists
|
||
// (Matching/NotMatching) — dborup wanted to see which specific nodes in
|
||
// e.g. Østjylland relay dk-oj (correctly "support" the area) and which
|
||
// don't, not just an aggregate count.
|
||
func computeScopeAdoptionByArea(nodes []nodeAreaScopeInput, areas map[string]AreaEntry, relayInfo map[string]RepeaterRelayInfo) []AreaScopeAdoption {
|
||
counts := make(map[string]*AreaScopeAdoption)
|
||
for _, n := range nodes {
|
||
keys := AreaKeysForPoint(n.Lat, n.Lon, areas)
|
||
if len(keys) == 0 {
|
||
continue
|
||
}
|
||
|
||
ownScope := strings.ToLower(strings.TrimPrefix(n.DefaultScope, "#"))
|
||
relayedRegions := make(map[string]bool)
|
||
if info, ok := relayInfo[n.PublicKey]; ok {
|
||
for _, r := range info.TransportedScopes {
|
||
relayedRegions[strings.ToLower(strings.TrimPrefix(r, "#"))] = true
|
||
}
|
||
}
|
||
hasAnyScope := ownScope != "" || len(relayedRegions) > 0
|
||
|
||
for _, key := range keys {
|
||
c, exists := counts[key]
|
||
if !exists {
|
||
a := areas[key]
|
||
c = &AreaScopeAdoption{AreaKey: key, Label: a.Label, RegionScopes: a.RegionScopes}
|
||
counts[key] = c
|
||
}
|
||
c.TotalNodes++
|
||
if hasAnyScope {
|
||
c.NodesWithAnyScope++
|
||
}
|
||
// Matching/NotMatching per-node lists only make sense when the
|
||
// area actually has region(s) to compare against — an area
|
||
// with no RegionScopes link has nothing to be "not matching".
|
||
if len(c.RegionScopes) > 0 {
|
||
var matchedScopes []string
|
||
for _, rs := range c.RegionScopes {
|
||
normalizedRegion := strings.ToLower(rs)
|
||
if ownScope == normalizedRegion || relayedRegions[normalizedRegion] {
|
||
matchedScopes = append(matchedScopes, rs)
|
||
}
|
||
}
|
||
if len(matchedScopes) > 0 {
|
||
c.NodesMatchingArea++
|
||
c.Matching = append(c.Matching, AreaScopeMatch{Name: n.Name, PublicKey: n.PublicKey, MatchedScopes: matchedScopes})
|
||
} else {
|
||
c.NotMatching = append(c.NotMatching, RepeaterRef{Name: n.Name, PublicKey: n.PublicKey})
|
||
}
|
||
}
|
||
}
|
||
}
|
||
result := make([]AreaScopeAdoption, 0, len(counts))
|
||
for _, c := range counts {
|
||
sort.Slice(c.Matching, func(i, j int) bool { return c.Matching[i].Name < c.Matching[j].Name })
|
||
sort.Slice(c.NotMatching, func(i, j int) bool { return c.NotMatching[i].Name < c.NotMatching[j].Name })
|
||
result = append(result, *c)
|
||
}
|
||
sort.Slice(result, func(i, j int) bool {
|
||
if result[i].TotalNodes != result[j].TotalNodes {
|
||
return result[i].TotalNodes > result[j].TotalNodes
|
||
}
|
||
return result[i].Label < result[j].Label
|
||
})
|
||
return result
|
||
}
|
||
|
||
// QueryMultiNodePackets returns transmissions referencing any of the given pubkeys.
|
||
func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) (*PacketResult, error) {
|
||
if len(pubkeys) == 0 {
|
||
return &PacketResult{Packets: []map[string]interface{}{}, Total: 0}, nil
|
||
}
|
||
if limit <= 0 {
|
||
limit = 50
|
||
}
|
||
if order == "" {
|
||
order = "DESC"
|
||
}
|
||
|
||
// Build IN(?, ?, ...) on the dedicated from_pubkey column (#1143):
|
||
// exact match, indexed lookup, no JSON substring scan.
|
||
var args []interface{}
|
||
placeholders := make([]string, 0, len(pubkeys))
|
||
for _, pk := range pubkeys {
|
||
resolved := db.resolveNodePubkey(pk)
|
||
args = append(args, resolved)
|
||
placeholders = append(placeholders, "?")
|
||
}
|
||
pkWhere := "t.from_pubkey IN (" + strings.Join(placeholders, ",") + ")"
|
||
|
||
var timeFilters []string
|
||
if since != "" {
|
||
timeFilters = append(timeFilters, "t.first_seen >= ?")
|
||
args = append(args, since)
|
||
}
|
||
if until != "" {
|
||
timeFilters = append(timeFilters, "t.first_seen <= ?")
|
||
args = append(args, until)
|
||
}
|
||
|
||
w := "WHERE " + pkWhere
|
||
if len(timeFilters) > 0 {
|
||
w += " AND " + strings.Join(timeFilters, " AND ")
|
||
}
|
||
|
||
var total int
|
||
db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM transmissions t %s", w), args...).Scan(&total)
|
||
|
||
selectCols, observerJoin := db.transmissionBaseSQL()
|
||
// #1345: order by ingest id (see QueryPackets comment above).
|
||
querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?",
|
||
selectCols, observerJoin, w, order)
|
||
|
||
qArgs := make([]interface{}, len(args))
|
||
copy(qArgs, args)
|
||
qArgs = append(qArgs, limit, offset)
|
||
|
||
rows, err := db.conn.Query(querySQL, qArgs...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
packets := make([]map[string]interface{}, 0)
|
||
for rows.Next() {
|
||
p := db.scanTransmissionRow(rows)
|
||
if p != nil {
|
||
packets = append(packets, p)
|
||
}
|
||
}
|
||
return &PacketResult{Packets: packets, Total: total}, nil
|
||
}
|
||
|
||
// --- Helpers ---
|
||
|
||
func scanPacketRow(rows *sql.Rows) map[string]interface{} {
|
||
var id int
|
||
var rawHex, ts, obsID, obsName, direction, hash, pathJSON, decodedJSON, createdAt sql.NullString
|
||
var snr, rssi sql.NullFloat64
|
||
var score, routeType, payloadType, payloadVersion sql.NullInt64
|
||
|
||
if err := rows.Scan(&id, &rawHex, &ts, &obsID, &obsName, &direction, &snr, &rssi, &score, &hash, &routeType, &payloadType, &payloadVersion, &pathJSON, &decodedJSON, &createdAt); err != nil {
|
||
return nil
|
||
}
|
||
return map[string]interface{}{
|
||
"id": id,
|
||
"raw_hex": nullStr(rawHex),
|
||
"timestamp": nullStr(ts),
|
||
"observer_id": nullStr(obsID),
|
||
"observer_name": nullStr(obsName),
|
||
"direction": nullStr(direction),
|
||
"snr": nullFloat(snr),
|
||
"rssi": nullFloat(rssi),
|
||
"score": nullInt(score),
|
||
"hash": nullStr(hash),
|
||
"route_type": nullInt(routeType),
|
||
"payload_type": nullInt(payloadType),
|
||
"payload_version": nullInt(payloadVersion),
|
||
"path_json": nullStr(pathJSON),
|
||
"decoded_json": nullStr(decodedJSON),
|
||
"created_at": nullStr(createdAt),
|
||
}
|
||
}
|
||
|
||
// scanNodeRow scans a node row. When hasDefaultScope is true the SELECT must
|
||
// include default_scope as the last column.
|
||
func (db *DB) scanNodeRow(rows *sql.Rows) map[string]interface{} {
|
||
var pk string
|
||
var name, role, lastSeen, firstSeen sql.NullString
|
||
var lat, lon sql.NullFloat64
|
||
var advertCount int
|
||
var batteryMv sql.NullInt64
|
||
var temperatureC sql.NullFloat64
|
||
var foreign sql.NullInt64
|
||
var defaultScope sql.NullString
|
||
|
||
scanArgs := []interface{}{&pk, &name, &role, &lat, &lon, &lastSeen, &firstSeen, &advertCount, &batteryMv, &temperatureC, &foreign}
|
||
if db.hasDefaultScope {
|
||
scanArgs = append(scanArgs, &defaultScope)
|
||
}
|
||
if err := rows.Scan(scanArgs...); err != nil {
|
||
return nil
|
||
}
|
||
m := map[string]interface{}{
|
||
"public_key": pk,
|
||
"name": nullStr(name),
|
||
"role": nullStr(role),
|
||
"lat": nullFloat(lat),
|
||
"lon": nullFloat(lon),
|
||
"last_seen": nullStr(lastSeen),
|
||
"first_seen": nullStr(firstSeen),
|
||
"advert_count": advertCount,
|
||
"last_heard": nullStr(lastSeen),
|
||
"hash_size": nil,
|
||
"hash_size_inconsistent": false,
|
||
"foreign": foreign.Valid && foreign.Int64 != 0,
|
||
}
|
||
if batteryMv.Valid {
|
||
m["battery_mv"] = int(batteryMv.Int64)
|
||
} else {
|
||
m["battery_mv"] = nil
|
||
}
|
||
if temperatureC.Valid {
|
||
m["temperature_c"] = temperatureC.Float64
|
||
} else {
|
||
m["temperature_c"] = nil
|
||
}
|
||
if db.hasDefaultScope {
|
||
m["default_scope"] = nullStr(defaultScope)
|
||
}
|
||
return m
|
||
}
|
||
|
||
func nullStr(ns sql.NullString) interface{} {
|
||
if ns.Valid {
|
||
return ns.String
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func nullStrVal(ns sql.NullString) string {
|
||
if ns.Valid {
|
||
return ns.String
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func nilIfEmpty(s string) interface{} {
|
||
if s == "" {
|
||
return nil
|
||
}
|
||
return s
|
||
}
|
||
|
||
func nullFloat(nf sql.NullFloat64) interface{} {
|
||
if nf.Valid {
|
||
return nf.Float64
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func nullInt(ni sql.NullInt64) interface{} {
|
||
if ni.Valid {
|
||
return int(ni.Int64)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// PruneOldPackets, PruneOldMetrics, and RemoveStaleObservers were
|
||
// removed in #1283 — they are write operations and now live on the
|
||
// ingestor's *Store (cmd/ingestor/maintenance.go and cmd/ingestor/db.go).
|
||
// The server is the read path; it must not hold the SQLite write lock.
|
||
|
||
// MetricsSample represents a single row from observer_metrics with computed deltas.
|
||
type MetricsSample struct {
|
||
Timestamp string `json:"timestamp"`
|
||
NoiseFloor *float64 `json:"noise_floor"`
|
||
TxAirSecs *int `json:"tx_air_secs,omitempty"`
|
||
RxAirSecs *int `json:"rx_air_secs,omitempty"`
|
||
RecvErrors *int `json:"recv_errors,omitempty"`
|
||
BatteryMv *int `json:"battery_mv"`
|
||
PacketsSent *int `json:"packets_sent,omitempty"`
|
||
PacketsRecv *int `json:"packets_recv,omitempty"`
|
||
TxAirtimePct *float64 `json:"tx_airtime_pct"`
|
||
RxAirtimePct *float64 `json:"rx_airtime_pct"`
|
||
RecvErrorRate *float64 `json:"recv_error_rate"`
|
||
IsReboot bool `json:"is_reboot_sample,omitempty"`
|
||
}
|
||
|
||
// rawMetricsSample is the raw DB row before delta computation.
|
||
type rawMetricsSample struct {
|
||
Timestamp string
|
||
NoiseFloor *float64
|
||
TxAirSecs *int
|
||
RxAirSecs *int
|
||
RecvErrors *int
|
||
BatteryMv *int
|
||
PacketsSent *int
|
||
PacketsRecv *int
|
||
}
|
||
|
||
// GetObserverMetrics returns time-series metrics with server-side delta computation.
|
||
// resolution: "5m" (raw), "1h", "1d"
|
||
// sampleIntervalSec: expected interval between samples (default 300)
|
||
func (db *DB) GetObserverMetrics(observerID, since, until, resolution string, sampleIntervalSec int) ([]MetricsSample, []string, error) {
|
||
if sampleIntervalSec <= 0 {
|
||
sampleIntervalSec = 300
|
||
}
|
||
|
||
// Build query based on resolution
|
||
var query string
|
||
args := []interface{}{observerID}
|
||
|
||
// Determine the effective bucket size for gap threshold scaling.
|
||
// For raw data (5m), use sampleIntervalSec. For aggregated resolutions,
|
||
// use the bucket duration so consecutive buckets aren't treated as gaps.
|
||
bucketSizeSec := sampleIntervalSec
|
||
switch resolution {
|
||
case "1h":
|
||
bucketSizeSec = 3600
|
||
// Use LAST value per bucket (latest timestamp) instead of MAX to preserve
|
||
// reboot semantics: if a device reboots mid-bucket, the last sample is the
|
||
// post-reboot baseline, not the pre-reboot high-water mark.
|
||
query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM (
|
||
SELECT
|
||
strftime('%Y-%m-%dT%H:00:00Z', timestamp) as ts,
|
||
noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv,
|
||
ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT%H:00:00Z', timestamp) ORDER BY timestamp DESC) as rn
|
||
FROM observer_metrics WHERE observer_id = ?`
|
||
case "1d":
|
||
bucketSizeSec = 86400
|
||
query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM (
|
||
SELECT
|
||
strftime('%Y-%m-%dT00:00:00Z', timestamp) as ts,
|
||
noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv,
|
||
ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT00:00:00Z', timestamp) ORDER BY timestamp DESC) as rn
|
||
FROM observer_metrics WHERE observer_id = ?`
|
||
default: // "5m" or raw
|
||
query = `SELECT timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv
|
||
FROM observer_metrics WHERE observer_id = ?`
|
||
}
|
||
|
||
if since != "" {
|
||
query += " AND timestamp >= ?"
|
||
args = append(args, since)
|
||
}
|
||
if until != "" {
|
||
query += " AND timestamp <= ?"
|
||
args = append(args, until)
|
||
}
|
||
|
||
switch resolution {
|
||
case "1h", "1d":
|
||
query += ") WHERE rn = 1 ORDER BY ts ASC"
|
||
default:
|
||
query += " ORDER BY timestamp ASC"
|
||
}
|
||
|
||
rows, err := db.conn.Query(query, args...)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
var raw []rawMetricsSample
|
||
for rows.Next() {
|
||
var s rawMetricsSample
|
||
if err := rows.Scan(&s.Timestamp, &s.NoiseFloor, &s.TxAirSecs, &s.RxAirSecs, &s.RecvErrors, &s.BatteryMv, &s.PacketsSent, &s.PacketsRecv); err != nil {
|
||
return nil, nil, err
|
||
}
|
||
raw = append(raw, s)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, nil, err
|
||
}
|
||
|
||
// Compute deltas between consecutive samples.
|
||
// bucketSizeSec determines gap threshold: for raw data it's sampleIntervalSec,
|
||
// for aggregated resolutions it's the bucket duration (3600 for 1h, 86400 for 1d).
|
||
return computeDeltas(raw, bucketSizeSec)
|
||
}
|
||
|
||
// computeDeltas computes per-interval rates from cumulative counters.
|
||
// Handles reboots (counter reset) and gaps (missing samples).
|
||
// bucketSizeSec is the expected interval between consecutive points
|
||
// (sampleInterval for raw data, bucket duration for aggregated resolutions).
|
||
func computeDeltas(raw []rawMetricsSample, bucketSizeSec int) ([]MetricsSample, []string, error) {
|
||
if len(raw) == 0 {
|
||
return nil, nil, nil
|
||
}
|
||
|
||
gapThreshold := float64(bucketSizeSec) * 2.0
|
||
result := make([]MetricsSample, 0, len(raw))
|
||
var reboots []string
|
||
|
||
for i, cur := range raw {
|
||
s := MetricsSample{
|
||
Timestamp: cur.Timestamp,
|
||
NoiseFloor: cur.NoiseFloor,
|
||
BatteryMv: cur.BatteryMv,
|
||
}
|
||
|
||
if i == 0 {
|
||
// First sample: no delta possible
|
||
result = append(result, s)
|
||
continue
|
||
}
|
||
|
||
prev := raw[i-1]
|
||
|
||
// Check for gap
|
||
curT, err1 := time.Parse(time.RFC3339, cur.Timestamp)
|
||
prevT, err2 := time.Parse(time.RFC3339, prev.Timestamp)
|
||
if err1 != nil || err2 != nil {
|
||
result = append(result, s)
|
||
continue
|
||
}
|
||
intervalSecs := curT.Sub(prevT).Seconds()
|
||
if intervalSecs > gapThreshold {
|
||
// Gap detected: insert null deltas (don't interpolate)
|
||
result = append(result, s)
|
||
continue
|
||
}
|
||
if intervalSecs <= 0 {
|
||
result = append(result, s)
|
||
continue
|
||
}
|
||
|
||
// Detect reboot: any cumulative counter decreased
|
||
isReboot := false
|
||
if cur.TxAirSecs != nil && prev.TxAirSecs != nil && *cur.TxAirSecs < *prev.TxAirSecs {
|
||
isReboot = true
|
||
}
|
||
if cur.RxAirSecs != nil && prev.RxAirSecs != nil && *cur.RxAirSecs < *prev.RxAirSecs {
|
||
isReboot = true
|
||
}
|
||
if cur.RecvErrors != nil && prev.RecvErrors != nil && *cur.RecvErrors < *prev.RecvErrors {
|
||
isReboot = true
|
||
}
|
||
if cur.PacketsSent != nil && prev.PacketsSent != nil && *cur.PacketsSent < *prev.PacketsSent {
|
||
isReboot = true
|
||
}
|
||
if cur.PacketsRecv != nil && prev.PacketsRecv != nil && *cur.PacketsRecv < *prev.PacketsRecv {
|
||
isReboot = true
|
||
}
|
||
|
||
if isReboot {
|
||
s.IsReboot = true
|
||
reboots = append(reboots, cur.Timestamp)
|
||
// Skip delta computation for reboot samples — use as new baseline
|
||
result = append(result, s)
|
||
continue
|
||
}
|
||
|
||
// Compute TX airtime percentage
|
||
if cur.TxAirSecs != nil && prev.TxAirSecs != nil {
|
||
delta := float64(*cur.TxAirSecs - *prev.TxAirSecs)
|
||
pct := (delta / intervalSecs) * 100.0
|
||
if pct < 0 {
|
||
pct = 0
|
||
}
|
||
if pct > 100 {
|
||
pct = 100
|
||
}
|
||
result_pct := math.Round(pct*100) / 100
|
||
s.TxAirtimePct = &result_pct
|
||
}
|
||
|
||
// Compute RX airtime percentage
|
||
if cur.RxAirSecs != nil && prev.RxAirSecs != nil {
|
||
delta := float64(*cur.RxAirSecs - *prev.RxAirSecs)
|
||
pct := (delta / intervalSecs) * 100.0
|
||
if pct < 0 {
|
||
pct = 0
|
||
}
|
||
if pct > 100 {
|
||
pct = 100
|
||
}
|
||
result_pct := math.Round(pct*100) / 100
|
||
s.RxAirtimePct = &result_pct
|
||
}
|
||
|
||
// Compute recv error rate
|
||
if cur.RecvErrors != nil && prev.RecvErrors != nil &&
|
||
cur.PacketsRecv != nil && prev.PacketsRecv != nil {
|
||
deltaErrors := float64(*cur.RecvErrors - *prev.RecvErrors)
|
||
deltaRecv := float64(*cur.PacketsRecv - *prev.PacketsRecv)
|
||
total := deltaRecv + deltaErrors
|
||
if total > 0 {
|
||
rate := (deltaErrors / total) * 100.0
|
||
rate = math.Round(rate*100) / 100
|
||
s.RecvErrorRate = &rate
|
||
}
|
||
}
|
||
|
||
result = append(result, s)
|
||
}
|
||
|
||
return result, reboots, nil
|
||
}
|
||
|
||
// MetricsSummaryRow holds summary data for one observer.
|
||
type MetricsSummaryRow struct {
|
||
ObserverID string `json:"observer_id"`
|
||
ObserverName *string `json:"observer_name"`
|
||
IATA string `json:"iata,omitempty"`
|
||
CurrentNF *float64 `json:"current_noise_floor"`
|
||
AvgNF *float64 `json:"avg_noise_floor_24h"`
|
||
MaxNF *float64 `json:"max_noise_floor_24h"`
|
||
CurrentBattMv *int `json:"battery_mv"`
|
||
SampleCount int `json:"sample_count"`
|
||
Sparkline []*float64 `json:"sparkline"`
|
||
}
|
||
|
||
// GetMetricsSummary returns a fleet summary of observer metrics within a time window.
|
||
// Uses a CTE with ROW_NUMBER to get latest values in a single pass (no correlated subqueries).
|
||
// Also returns sparkline data (noise_floor time series) per observer.
|
||
func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) {
|
||
query := `
|
||
WITH ranked AS (
|
||
SELECT observer_id, noise_floor, battery_mv,
|
||
ROW_NUMBER() OVER (PARTITION BY observer_id ORDER BY timestamp DESC) as rn
|
||
FROM observer_metrics
|
||
WHERE timestamp >= ?
|
||
)
|
||
SELECT m.observer_id, o.name, COALESCE(o.iata, '') as iata,
|
||
r.noise_floor as current_nf,
|
||
AVG(m.noise_floor) as avg_nf,
|
||
MAX(m.noise_floor) as max_nf,
|
||
r.battery_mv as current_batt,
|
||
COUNT(*) as sample_count
|
||
FROM observer_metrics m
|
||
LEFT JOIN observers o ON o.id = m.observer_id
|
||
LEFT JOIN ranked r ON r.observer_id = m.observer_id AND r.rn = 1
|
||
WHERE m.timestamp >= ?
|
||
GROUP BY m.observer_id
|
||
ORDER BY max_nf DESC
|
||
`
|
||
rows, err := db.conn.Query(query, since, since)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
var result []MetricsSummaryRow
|
||
for rows.Next() {
|
||
var s MetricsSummaryRow
|
||
if err := rows.Scan(&s.ObserverID, &s.ObserverName, &s.IATA, &s.CurrentNF, &s.AvgNF, &s.MaxNF, &s.CurrentBattMv, &s.SampleCount); err != nil {
|
||
return nil, err
|
||
}
|
||
result = append(result, s)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Fetch sparkline data (noise_floor series) for all observers in one query
|
||
if len(result) > 0 {
|
||
sparkQuery := `SELECT observer_id, noise_floor FROM observer_metrics
|
||
WHERE timestamp >= ? ORDER BY observer_id, timestamp ASC`
|
||
sparkRows, err := db.conn.Query(sparkQuery, since)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer sparkRows.Close()
|
||
|
||
sparkMap := make(map[string][]*float64)
|
||
for sparkRows.Next() {
|
||
var oid string
|
||
var nf *float64
|
||
if err := sparkRows.Scan(&oid, &nf); err != nil {
|
||
return nil, err
|
||
}
|
||
sparkMap[oid] = append(sparkMap[oid], nf)
|
||
}
|
||
if err := sparkRows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
for i := range result {
|
||
if s, ok := sparkMap[result[i].ObserverID]; ok {
|
||
result[i].Sparkline = s
|
||
}
|
||
}
|
||
}
|
||
|
||
return result, nil
|
||
}
|
||
|
||
// (PruneOldMetrics / RemoveStaleObservers removed in #1283 — see note
|
||
// above the MetricsSample type. Ingestor owns these writes now.)
|
||
|
||
// TouchNodeLastSeen updates last_seen for a node identified by full public key.
|
||
// Only updates if the new timestamp is newer than the existing value (or NULL).
|
||
// Returns nil even if no rows are affected (node doesn't exist).
|
||
func (db *DB) TouchNodeLastSeen(pubkey string, timestamp string) error {
|
||
_, err := db.conn.Exec(
|
||
"UPDATE nodes SET last_seen = ? WHERE public_key = ? AND (last_seen IS NULL OR last_seen < ?)",
|
||
timestamp, pubkey, timestamp,
|
||
)
|
||
return err
|
||
}
|
||
|
||
// GetDroppedPackets returns recently dropped packets, newest first.
|
||
func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map[string]interface{}, error) {
|
||
if limit <= 0 || limit > 500 {
|
||
limit = 100
|
||
}
|
||
query := `SELECT id, hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name, dropped_at FROM dropped_packets`
|
||
var conditions []string
|
||
var args []interface{}
|
||
if observerID != "" {
|
||
conditions = append(conditions, "observer_id = ?")
|
||
args = append(args, observerID)
|
||
}
|
||
if nodePubkey != "" {
|
||
conditions = append(conditions, "node_pubkey = ?")
|
||
args = append(args, nodePubkey)
|
||
}
|
||
if len(conditions) > 0 {
|
||
query += " WHERE " + strings.Join(conditions, " AND ")
|
||
}
|
||
query += " ORDER BY dropped_at DESC LIMIT ?"
|
||
args = append(args, limit)
|
||
|
||
rows, err := db.conn.Query(query, args...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
var results []map[string]interface{}
|
||
for rows.Next() {
|
||
var id int
|
||
var hash, rawHex, reason, obsID, obsName, pubkey, name, droppedAt sql.NullString
|
||
if err := rows.Scan(&id, &hash, &rawHex, &reason, &obsID, &obsName, &pubkey, &name, &droppedAt); err != nil {
|
||
continue
|
||
}
|
||
row := map[string]interface{}{
|
||
"id": id,
|
||
"hash": nullStr(hash),
|
||
"reason": nullStr(reason),
|
||
"observer_id": nullStr(obsID),
|
||
"observer_name": nullStr(obsName),
|
||
"node_pubkey": nullStr(pubkey),
|
||
"node_name": nullStr(name),
|
||
"dropped_at": nullStr(droppedAt),
|
||
}
|
||
// Only include raw_hex if explicitly requested (it's large)
|
||
if rawHex.Valid {
|
||
row["raw_hex"] = rawHex.String
|
||
}
|
||
results = append(results, row)
|
||
}
|
||
if results == nil {
|
||
results = []map[string]interface{}{}
|
||
}
|
||
return results, nil
|
||
}
|
||
|
||
// GetNodePubkeysInArea returns public keys of nodes whose GPS coordinates
|
||
// fall inside the given area polygon or bounding box.
|
||
func (db *DB) GetNodePubkeysInArea(entry AreaEntry) ([]string, error) {
|
||
rows, err := db.conn.Query("SELECT public_key, lat, lon FROM nodes WHERE lat IS NOT NULL AND lon IS NOT NULL")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
gf := &geofilter.Config{
|
||
Polygon: entry.Polygon,
|
||
LatMin: entry.LatMin,
|
||
LatMax: entry.LatMax,
|
||
LonMin: entry.LonMin,
|
||
LonMax: entry.LonMax,
|
||
}
|
||
|
||
var result []string
|
||
for rows.Next() {
|
||
var pk string
|
||
var lat, lon sql.NullFloat64
|
||
if err := rows.Scan(&pk, &lat, &lon); err != nil {
|
||
continue
|
||
}
|
||
if !lat.Valid || !lon.Valid {
|
||
continue
|
||
}
|
||
// Skip (0,0) — PassesFilter allows it but these nodes have no real GPS.
|
||
if lat.Float64 == 0 && lon.Float64 == 0 {
|
||
continue
|
||
}
|
||
if geofilter.PassesFilter(lat.Float64, lon.Float64, gf) {
|
||
result = append(result, pk)
|
||
}
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// GetSignatureDropCount returns the total number of dropped packets.
|
||
func (db *DB) GetSignatureDropCount() int64 {
|
||
var count int64
|
||
// Table may not exist yet if ingestor hasn't run the migration
|
||
err := db.conn.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&count)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
return count
|
||
}
|
||
|
||
func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) {
|
||
if !db.hasScopeName {
|
||
return nil, fmt.Errorf("scope_name column not present — run ingestor to apply migrations")
|
||
}
|
||
|
||
var since string
|
||
var bucketExpr string
|
||
switch window {
|
||
case "1h":
|
||
since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339)
|
||
// 5-minute buckets
|
||
bucketExpr = `strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'`
|
||
case "7d":
|
||
since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||
// 6-hour buckets
|
||
bucketExpr = `strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`
|
||
default: // "24h"
|
||
window = "24h"
|
||
since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||
// 1-hour buckets
|
||
bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', first_seen)`
|
||
}
|
||
|
||
resp := &ScopeStatsResponse{Window: window}
|
||
|
||
// Summary counts
|
||
row := db.conn.QueryRow(`
|
||
SELECT
|
||
COUNT(*) AS transport_total,
|
||
COUNT(scope_name) AS scoped,
|
||
COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped,
|
||
COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope
|
||
FROM transmissions
|
||
WHERE `+routeTypeTransportSQL+` AND first_seen >= ?
|
||
`, since)
|
||
if err := row.Scan(
|
||
&resp.Summary.TransportTotal,
|
||
&resp.Summary.Scoped,
|
||
&resp.Summary.Unscoped,
|
||
&resp.Summary.UnknownScope,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scope summary query: %w", err)
|
||
}
|
||
|
||
// #1838: non-transport routes (FLOOD=1, DIRECT=2) never carry
|
||
// transport_code_1 per MeshCore protocol, so they are inherently unscoped.
|
||
// Fold their count into Summary.Unscoped so the analytics denominator
|
||
// reflects total-observed-transmissions rather than only transport-eligible.
|
||
var nonTransportUnscoped int
|
||
if err := db.conn.QueryRow(`
|
||
SELECT COUNT(*) FROM transmissions
|
||
WHERE `+routeTypeNonTransportSQL+` AND first_seen >= ?
|
||
`, since).Scan(&nonTransportUnscoped); err != nil {
|
||
return nil, fmt.Errorf("scope non-transport count query: %w", err)
|
||
}
|
||
resp.Summary.Unscoped += nonTransportUnscoped
|
||
|
||
// Per-region counts (named regions only)
|
||
rows, err := db.conn.Query(`
|
||
SELECT scope_name, COUNT(*) AS cnt
|
||
FROM transmissions
|
||
WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= ?
|
||
GROUP BY scope_name
|
||
ORDER BY cnt DESC
|
||
`, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("scope byRegion query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var rc ScopeRegionCount
|
||
if rows.Scan(&rc.Name, &rc.Count) == nil {
|
||
resp.ByRegion = append(resp.ByRegion, rc)
|
||
}
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, fmt.Errorf("scope byRegion iteration: %w", err)
|
||
}
|
||
if resp.ByRegion == nil {
|
||
resp.ByRegion = []ScopeRegionCount{}
|
||
}
|
||
|
||
// Time series
|
||
tsQuery := fmt.Sprintf(`
|
||
SELECT %s AS bucket,
|
||
COUNT(scope_name) AS scoped,
|
||
SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped
|
||
FROM transmissions
|
||
WHERE `+routeTypeTransportSQL+` AND first_seen >= ?
|
||
GROUP BY bucket
|
||
ORDER BY bucket
|
||
`, bucketExpr)
|
||
tsRows, err := db.conn.Query(tsQuery, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("scope timeseries query: %w", err)
|
||
}
|
||
defer tsRows.Close()
|
||
for tsRows.Next() {
|
||
var pt ScopeTimePoint
|
||
if tsRows.Scan(&pt.T, &pt.Scoped, &pt.Unscoped) == nil {
|
||
resp.TimeSeries = append(resp.TimeSeries, pt)
|
||
}
|
||
}
|
||
if err := tsRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("scope timeseries iteration: %w", err)
|
||
}
|
||
if resp.TimeSeries == nil {
|
||
resp.TimeSeries = []ScopeTimePoint{}
|
||
}
|
||
|
||
// Hour-of-day activity per region: same named-region set as ByRegion,
|
||
// but bucketed by hour-of-day (0-23, UTC) instead of chronological
|
||
// time — answers "when during a typical day is this region active"
|
||
// rather than "how did volume change over the window". Aggregated
|
||
// across every day in the window, so this reads best on 7d (a single
|
||
// 1h/24h window won't show a meaningful daily shape).
|
||
hourRows, err := db.conn.Query(`
|
||
SELECT scope_name, CAST(strftime('%H', first_seen) AS INTEGER) AS hour, COUNT(*) AS cnt
|
||
FROM transmissions
|
||
WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= ?
|
||
GROUP BY scope_name, hour
|
||
`, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("scope hourly activity query: %w", err)
|
||
}
|
||
defer hourRows.Close()
|
||
hourly := make(map[string]*ScopeHourlyActivity)
|
||
var hourlyOrder []string
|
||
for hourRows.Next() {
|
||
var region string
|
||
var hour, cnt int
|
||
if hourRows.Scan(®ion, &hour, &cnt) != nil {
|
||
continue
|
||
}
|
||
if hour < 0 || hour > 23 {
|
||
continue
|
||
}
|
||
ha, ok := hourly[region]
|
||
if !ok {
|
||
ha = &ScopeHourlyActivity{Region: region}
|
||
hourly[region] = ha
|
||
hourlyOrder = append(hourlyOrder, region)
|
||
}
|
||
ha.Hours[hour] = cnt
|
||
}
|
||
if err := hourRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("scope hourly activity iteration: %w", err)
|
||
}
|
||
resp.HourlyActivityByRegion = make([]ScopeHourlyActivity, 0, len(hourlyOrder))
|
||
for _, region := range hourlyOrder {
|
||
resp.HourlyActivityByRegion = append(resp.HourlyActivityByRegion, *hourly[region])
|
||
}
|
||
sort.Slice(resp.HourlyActivityByRegion, func(i, j int) bool {
|
||
return resp.HourlyActivityByRegion[i].Region < resp.HourlyActivityByRegion[j].Region
|
||
})
|
||
|
||
return resp, nil
|
||
}
|
||
|
||
// GetHopDepthAnalytics answers, network-wide over the given window, two
|
||
// questions that share the same expensive "walk every resolved relay path"
|
||
// pass — see HopDepthAnalyticsResponse's doc comment for the full
|
||
// rationale. "Scoped" here is derived purely from route_type
|
||
// (TRANSPORT_FLOOD=0/TRANSPORT_DIRECT=3 vs FLOOD=1/DIRECT=2, the same
|
||
// convention used throughout this codebase — e.g. computeHopAnalyticsTransport),
|
||
// not the scope_name column, so this works even on schemas predating #899.
|
||
//
|
||
// A transmission can have resolved_path stored on more than one
|
||
// observation; this keeps whichever has the MOST entries per transmission
|
||
// (a proxy for "most complete"), matching fetchResolvedPathForTxBest's
|
||
// "longest wins" selection without needing per-tx round trips.
|
||
func (db *DB) GetHopDepthAnalytics(window string) (*HopDepthAnalyticsResponse, error) {
|
||
var since string
|
||
var bucketExpr string
|
||
switch window {
|
||
case "1h":
|
||
since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339)
|
||
// 5-minute buckets -- same bucketing as GetScopeStats' TimeSeries.
|
||
bucketExpr = `strftime('%Y-%m-%dT%H:', t.first_seen) || printf('%02d', (CAST(strftime('%M', t.first_seen) AS INTEGER) / 5) * 5) || ':00Z'`
|
||
case "7d":
|
||
since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||
// 6-hour buckets
|
||
bucketExpr = `strftime('%Y-%m-%dT', t.first_seen) || printf('%02d', (CAST(strftime('%H', t.first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`
|
||
default:
|
||
window = "24h"
|
||
since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||
// 1-hour buckets
|
||
bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', t.first_seen)`
|
||
}
|
||
|
||
rows, err := db.conn.Query(`
|
||
SELECT t.id, t.route_type, t.payload_type, o.resolved_path, `+bucketExpr+` AS bucket
|
||
FROM transmissions t
|
||
JOIN observations o ON o.transmission_id = t.id
|
||
WHERE t.first_seen > ? AND o.resolved_path IS NOT NULL AND o.resolved_path != ''`, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("hop depth analytics query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
type txInfo struct {
|
||
routeType sql.NullInt64
|
||
payloadType sql.NullInt64
|
||
bestPath []*string
|
||
bucket string
|
||
}
|
||
byTx := make(map[int]*txInfo)
|
||
for rows.Next() {
|
||
var txID int
|
||
var routeType, payloadType sql.NullInt64
|
||
var rpJSON, bucket string
|
||
if err := rows.Scan(&txID, &routeType, &payloadType, &rpJSON, &bucket); err != nil {
|
||
continue
|
||
}
|
||
rp := unmarshalResolvedPath(rpJSON)
|
||
if len(rp) == 0 {
|
||
continue
|
||
}
|
||
cur, ok := byTx[txID]
|
||
if !ok {
|
||
byTx[txID] = &txInfo{routeType: routeType, payloadType: payloadType, bestPath: rp, bucket: bucket}
|
||
continue
|
||
}
|
||
if len(rp) > len(cur.bestPath) {
|
||
cur.bestPath = rp
|
||
}
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, fmt.Errorf("hop depth analytics iteration: %w", err)
|
||
}
|
||
|
||
scopedBuckets := map[int]int{}
|
||
unscopedBuckets := map[int]int{}
|
||
repeaterHops := map[string][]int{}
|
||
// Same scoped/unscoped hop-index tally as scopedBuckets/unscopedBuckets
|
||
// above, but split by time bucket too -- feeds TimeSeries, answering
|
||
// "is containment getting better or worse over the window" instead of
|
||
// just a single window-wide snapshot.
|
||
scopedByTimeBucket := map[string]map[int]int{}
|
||
unscopedByTimeBucket := map[string]map[int]int{}
|
||
|
||
for _, info := range byTx {
|
||
if !info.routeType.Valid {
|
||
continue
|
||
}
|
||
rt := int(info.routeType.Int64)
|
||
isFlood := rt == routeTypeFlood || rt == RouteTransportFlood
|
||
if !isFlood {
|
||
continue
|
||
}
|
||
scoped := rt == RouteTransportFlood
|
||
isAdvert := info.payloadType.Valid && int(info.payloadType.Int64) == payloadTypeAdvert
|
||
isUnscopedFlood := rt == routeTypeFlood && !isAdvert
|
||
|
||
for idx, pk := range info.bestPath {
|
||
if scoped {
|
||
scopedBuckets[idx]++
|
||
if scopedByTimeBucket[info.bucket] == nil {
|
||
scopedByTimeBucket[info.bucket] = map[int]int{}
|
||
}
|
||
scopedByTimeBucket[info.bucket][idx]++
|
||
} else {
|
||
unscopedBuckets[idx]++
|
||
if unscopedByTimeBucket[info.bucket] == nil {
|
||
unscopedByTimeBucket[info.bucket] = map[int]int{}
|
||
}
|
||
unscopedByTimeBucket[info.bucket][idx]++
|
||
}
|
||
if isUnscopedFlood && pk != nil && *pk != "" {
|
||
repeaterHops[*pk] = append(repeaterHops[*pk], idx)
|
||
}
|
||
}
|
||
}
|
||
|
||
toSortedBuckets := func(m map[int]int) []HopDepthBucket {
|
||
out := make([]HopDepthBucket, 0, len(m))
|
||
for hops, count := range m {
|
||
out = append(out, HopDepthBucket{Hops: hops, Count: count})
|
||
}
|
||
sort.Slice(out, func(i, j int) bool { return out[i].Hops < out[j].Hops })
|
||
return out
|
||
}
|
||
|
||
// medianHopFromCounts mirrors the frontend's hopDepthBucketStats: the
|
||
// smallest hop value whose cumulative count reaches half the total --
|
||
// a histogram-based median consistent with how ScopedHopDepth/
|
||
// UnscopedHopDepth's medians are read client-side, rather than
|
||
// introducing a second, subtly different median definition here.
|
||
medianHopFromCounts := func(counts map[int]int) *int {
|
||
total := 0
|
||
for _, c := range counts {
|
||
total += c
|
||
}
|
||
if total == 0 {
|
||
return nil
|
||
}
|
||
hops := make([]int, 0, len(counts))
|
||
for h := range counts {
|
||
hops = append(hops, h)
|
||
}
|
||
sort.Ints(hops)
|
||
cum := 0
|
||
for _, h := range hops {
|
||
cum += counts[h]
|
||
if cum*2 >= total {
|
||
v := h
|
||
return &v
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
bucketSet := map[string]bool{}
|
||
for b := range scopedByTimeBucket {
|
||
bucketSet[b] = true
|
||
}
|
||
for b := range unscopedByTimeBucket {
|
||
bucketSet[b] = true
|
||
}
|
||
timeBuckets := make([]string, 0, len(bucketSet))
|
||
for b := range bucketSet {
|
||
timeBuckets = append(timeBuckets, b)
|
||
}
|
||
sort.Strings(timeBuckets)
|
||
timeSeries := make([]HopDepthTimePoint, 0, len(timeBuckets))
|
||
for _, b := range timeBuckets {
|
||
timeSeries = append(timeSeries, HopDepthTimePoint{
|
||
T: b,
|
||
ScopedMedianHop: medianHopFromCounts(scopedByTimeBucket[b]),
|
||
UnscopedMedianHop: medianHopFromCounts(unscopedByTimeBucket[b]),
|
||
})
|
||
}
|
||
|
||
// Only repeater/room nodes are meaningful here (matches the Foreign
|
||
// Traffic tab's existing "Repeaters Relaying Unscoped Traffic" role
|
||
// filter) -- look up name/role for the pubkeys that actually relayed
|
||
// unscoped flood traffic in this window, rather than every known node.
|
||
pubkeys := make([]string, 0, len(repeaterHops))
|
||
for pk := range repeaterHops {
|
||
pubkeys = append(pubkeys, pk)
|
||
}
|
||
names, roles := db.namesAndRolesForPubkeys(pubkeys)
|
||
|
||
unscopedByRepeater := make([]RepeaterUnscopedHopDepth, 0, len(repeaterHops))
|
||
for pk, hops := range repeaterHops {
|
||
if roles[pk] != "repeater" && roles[pk] != "room" {
|
||
continue
|
||
}
|
||
sort.Ints(hops)
|
||
n := len(hops)
|
||
median := float64(hops[n/2])
|
||
if n%2 == 0 {
|
||
median = float64(hops[n/2-1]+hops[n/2]) / 2
|
||
}
|
||
name := names[pk]
|
||
if name == "" {
|
||
name = pk
|
||
}
|
||
unscopedByRepeater = append(unscopedByRepeater, RepeaterUnscopedHopDepth{
|
||
PublicKey: pk,
|
||
Name: name,
|
||
Count: n,
|
||
MinHops: hops[0],
|
||
MedianHops: median,
|
||
MaxHops: hops[n-1],
|
||
})
|
||
}
|
||
sort.Slice(unscopedByRepeater, func(i, j int) bool { return unscopedByRepeater[i].Count > unscopedByRepeater[j].Count })
|
||
|
||
return &HopDepthAnalyticsResponse{
|
||
Window: window,
|
||
ScopedHopDepth: toSortedBuckets(scopedBuckets),
|
||
UnscopedHopDepth: toSortedBuckets(unscopedBuckets),
|
||
UnscopedByRepeater: unscopedByRepeater,
|
||
TimeSeries: timeSeries,
|
||
}, nil
|
||
}
|
||
|
||
// namesAndRolesForPubkeys bulk-looks-up name/role for a set of pubkeys,
|
||
// chunked to stay under SQLite's parameter limit. Missing pubkeys are
|
||
// simply absent from the returned maps.
|
||
func (db *DB) namesAndRolesForPubkeys(pubkeys []string) (names, roles map[string]string) {
|
||
names = make(map[string]string, len(pubkeys))
|
||
roles = make(map[string]string, len(pubkeys))
|
||
if len(pubkeys) == 0 {
|
||
return names, roles
|
||
}
|
||
const chunkSize = 499
|
||
for start := 0; start < len(pubkeys); start += chunkSize {
|
||
end := start + chunkSize
|
||
if end > len(pubkeys) {
|
||
end = len(pubkeys)
|
||
}
|
||
chunk := pubkeys[start:end]
|
||
placeholders := make([]byte, 0, len(chunk)*2)
|
||
args := make([]interface{}, len(chunk))
|
||
for i, pk := range chunk {
|
||
if i > 0 {
|
||
placeholders = append(placeholders, ',')
|
||
}
|
||
placeholders = append(placeholders, '?')
|
||
args[i] = pk
|
||
}
|
||
query := "SELECT public_key, name, role FROM nodes WHERE public_key IN (" + string(placeholders) + ")"
|
||
rows, err := db.conn.Query(query, args...)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
for rows.Next() {
|
||
var pk string
|
||
var name, role sql.NullString
|
||
if err := rows.Scan(&pk, &name, &role); err != nil {
|
||
continue
|
||
}
|
||
names[pk] = name.String
|
||
roles[pk] = role.String
|
||
}
|
||
rows.Close()
|
||
}
|
||
return names, roles
|
||
}
|
||
|
||
// gpsByPubkeysExact bulk-resolves GPS positions for a set of FULL pubkeys
|
||
// via an exact match -- unlike the path-hop prefix machinery (buildPrefixMap
|
||
// /resolveEntryPointArea), which only indexes repeater/room-server roles as
|
||
// path-hop candidates, this has no role filter: any node type qualifies,
|
||
// since a hearing station's own position doesn't depend on whether it could
|
||
// ever appear as a relay hop in someone else's path.
|
||
func (db *DB) gpsByPubkeysExact(pubkeys []string) map[string][2]float64 {
|
||
result := make(map[string][2]float64, len(pubkeys))
|
||
if len(pubkeys) == 0 {
|
||
return result
|
||
}
|
||
const chunkSize = 499
|
||
for start := 0; start < len(pubkeys); start += chunkSize {
|
||
end := start + chunkSize
|
||
if end > len(pubkeys) {
|
||
end = len(pubkeys)
|
||
}
|
||
chunk := pubkeys[start:end]
|
||
placeholders := make([]byte, 0, len(chunk)*2)
|
||
args := make([]interface{}, len(chunk))
|
||
for i, pk := range chunk {
|
||
if i > 0 {
|
||
placeholders = append(placeholders, ',')
|
||
}
|
||
placeholders = append(placeholders, '?')
|
||
args[i] = pk
|
||
}
|
||
query := "SELECT public_key, lat, lon FROM nodes WHERE public_key IN (" + string(placeholders) + ")"
|
||
rows, err := db.conn.Query(query, args...)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
for rows.Next() {
|
||
var pk string
|
||
var lat, lon sql.NullFloat64
|
||
if rows.Scan(&pk, &lat, &lon) != nil {
|
||
continue
|
||
}
|
||
// (0,0) is the ocean off Ghana, not a real fix -- same
|
||
// exclusion GetPacketPath/packetSpreadStats apply.
|
||
if lat.Valid && lon.Valid && !(lat.Float64 == 0 && lon.Float64 == 0) {
|
||
result[pk] = [2]float64{lat.Float64, lon.Float64}
|
||
}
|
||
}
|
||
rows.Close()
|
||
}
|
||
return result
|
||
}
|
||
|
||
// GetChannelMessageScopeStats narrows the scoped/unscoped/unknown question
|
||
// to channel chat specifically (payload_type=5), for the given window.
|
||
// Unlike GetScopeStats' TransportTotal (route_type 0/3 only), TotalMessages
|
||
// here covers ALL route types — most channel chat is plain FLOOD, so
|
||
// restricting to transport routes would answer a different question than
|
||
// "how many of our channel messages are scoped".
|
||
func (db *DB) GetChannelMessageScopeStats(window string) (*ChannelScopeStats, error) {
|
||
if !db.hasScopeName {
|
||
return nil, fmt.Errorf("scope_name column not present — run ingestor to apply migrations")
|
||
}
|
||
|
||
var since string
|
||
switch window {
|
||
case "1h":
|
||
since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339)
|
||
case "7d":
|
||
since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||
default:
|
||
since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||
}
|
||
|
||
stats := &ChannelScopeStats{}
|
||
row := db.conn.QueryRow(`
|
||
SELECT
|
||
COUNT(*) AS transport_total,
|
||
COUNT(scope_name) AS scoped,
|
||
COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped,
|
||
COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope
|
||
FROM transmissions
|
||
WHERE payload_type = 5 AND `+routeTypeTransportSQL+` AND first_seen >= ?
|
||
`, since)
|
||
var transportTotal int
|
||
if err := row.Scan(&transportTotal, &stats.Scoped, &stats.Unscoped, &stats.UnknownScope); err != nil {
|
||
return nil, fmt.Errorf("channel scope summary query: %w", err)
|
||
}
|
||
|
||
// Non-transport channel messages (plain FLOOD/DIRECT) never carry a
|
||
// scope per MeshCore protocol — fold into Unscoped, mirroring #1838.
|
||
var nonTransportCount int
|
||
if err := db.conn.QueryRow(`
|
||
SELECT COUNT(*) FROM transmissions
|
||
WHERE payload_type = 5 AND `+routeTypeNonTransportSQL+` AND first_seen >= ?
|
||
`, since).Scan(&nonTransportCount); err != nil {
|
||
return nil, fmt.Errorf("channel scope non-transport count query: %w", err)
|
||
}
|
||
stats.Unscoped += nonTransportCount
|
||
stats.TotalMessages = transportTotal + nonTransportCount
|
||
|
||
return stats, nil
|
||
}
|
||
|
||
// GetChannelScopeAdoption breaks the channel-messages-only scoped/unscoped
|
||
// question (see GetChannelMessageScopeStats) down PER CHANNEL — which
|
||
// specific channels (#test, #wardriving, ...) actually use region scoping
|
||
// vs which never do. Ordered by message volume; uncapped.
|
||
//
|
||
// Cardinality isn't a hard bound like the 1-byte hash space alone would
|
||
// suggest: encrypted channels ARE bounded to 256 'enc_%02x' buckets, but
|
||
// plain-text CHAN channels use the free-form channel name string
|
||
// (ingestor/db.go's `json_extract(decoded_json, '$.channel')`), so in
|
||
// principle a mesh with many ad-hoc named channels could grow this
|
||
// unbounded. In practice payload_type=5 rows within a single window stay
|
||
// small enough that this hasn't needed a cap.
|
||
func (db *DB) GetChannelScopeAdoption(window string) ([]ChannelScopeAdoption, error) {
|
||
if !db.hasScopeName {
|
||
return nil, fmt.Errorf("scope_name column not present — run ingestor to apply migrations")
|
||
}
|
||
|
||
var since string
|
||
switch window {
|
||
case "1h":
|
||
since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339)
|
||
case "7d":
|
||
since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||
default:
|
||
since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||
}
|
||
|
||
rows, err := db.conn.Query(`
|
||
SELECT
|
||
COALESCE(NULLIF(channel_hash, ''), '(unknown channel)') AS channel,
|
||
COUNT(*) AS total,
|
||
COALESCE(SUM(CASE WHEN `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' THEN 1 ELSE 0 END), 0) AS scoped,
|
||
COALESCE(SUM(CASE WHEN `+routeTypeTransportSQL+` AND scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope
|
||
FROM transmissions
|
||
WHERE payload_type = 5 AND first_seen >= ?
|
||
GROUP BY channel
|
||
ORDER BY total DESC
|
||
`, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("channel scope adoption query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
result := make([]ChannelScopeAdoption, 0)
|
||
for rows.Next() {
|
||
var ca ChannelScopeAdoption
|
||
if err := rows.Scan(&ca.Channel, &ca.TotalMessages, &ca.Scoped, &ca.UnknownScope); err != nil {
|
||
continue
|
||
}
|
||
ca.Unscoped = ca.TotalMessages - ca.Scoped - ca.UnknownScope
|
||
result = append(result, ca)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, fmt.Errorf("channel scope adoption iteration: %w", err)
|
||
}
|
||
|
||
regionsByChannel, err := db.getChannelScopeRegions(since)
|
||
if err != nil {
|
||
// Non-fatal: the adoption counts above are still useful without
|
||
// the per-channel region breakdown.
|
||
log.Printf("WARN getChannelScopeRegions: %v", err)
|
||
} else {
|
||
for i := range result {
|
||
result[i].Regions = regionsByChannel[result[i].Channel]
|
||
}
|
||
}
|
||
|
||
return result, nil
|
||
}
|
||
|
||
// getChannelScopeRegions answers "which regions" for GetChannelScopeAdoption's
|
||
// "how many scoped messages" — for each channel, which distinct scope_name
|
||
// values have actually been seen on its scoped messages, most-used first.
|
||
func (db *DB) getChannelScopeRegions(since string) (map[string][]string, error) {
|
||
rows, err := db.conn.Query(`
|
||
SELECT
|
||
COALESCE(NULLIF(channel_hash, ''), '(unknown channel)') AS channel,
|
||
scope_name,
|
||
COUNT(*) AS cnt
|
||
FROM transmissions
|
||
WHERE payload_type = 5 AND first_seen >= ? AND `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != ''
|
||
GROUP BY channel, scope_name
|
||
ORDER BY channel, cnt DESC
|
||
`, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("channel scope regions query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
result := make(map[string][]string)
|
||
for rows.Next() {
|
||
var channel, region string
|
||
var cnt int
|
||
if err := rows.Scan(&channel, ®ion, &cnt); err != nil {
|
||
continue
|
||
}
|
||
result[channel] = append(result[channel], region)
|
||
}
|
||
return result, rows.Err()
|
||
}
|
||
|
||
// GetWardrivingStats aggregates activity on the given channel (normally
|
||
// "#wardriving") over the requested window: message volume over time, who's
|
||
// actively sending, which repeater first relayed each message (raw hash
|
||
// prefixes — the caller resolves names via /api/resolve-hops), which
|
||
// observer stations actually heard the traffic, signal quality (SNR/RSSI)
|
||
// over the same time buckets as the activity series, each sender's messages
|
||
// grouped into distinct sessions/runs (see buildWardrivingSessions), and any
|
||
// senders who explicitly shared their own position (see
|
||
// detectWardrivingGPSShares). See WardrivingObserverCoverage
|
||
// doc for why observer coverage — not sender GPS — is the reliable half of
|
||
// a "where did this reach" picture: MeshMapper's #wardriving messages carry
|
||
// an anonymous per-session token by default, not the sender's live
|
||
// coordinates (those go to MeshMapper's own server via a separate API call
|
||
// we have no visibility into).
|
||
func (db *DB) GetWardrivingStats(window, channel string) (*WardrivingStatsResponse, error) {
|
||
var since string
|
||
var bucketExpr string
|
||
switch window {
|
||
case "1h":
|
||
since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339)
|
||
bucketExpr = `strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'`
|
||
case "7d":
|
||
since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||
bucketExpr = `strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`
|
||
default:
|
||
window = "24h"
|
||
since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||
bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', first_seen)`
|
||
}
|
||
|
||
resp := &WardrivingStatsResponse{Window: window, Channel: channel}
|
||
|
||
if err := db.conn.QueryRow(
|
||
`SELECT COUNT(*) FROM transmissions WHERE channel_hash = ? AND payload_type = 5 AND first_seen >= ?`,
|
||
channel, since,
|
||
).Scan(&resp.TotalMessages); err != nil {
|
||
return nil, fmt.Errorf("wardriving total query: %w", err)
|
||
}
|
||
|
||
tsQuery := fmt.Sprintf(`
|
||
SELECT %s AS bucket, COUNT(*) AS cnt
|
||
FROM transmissions
|
||
WHERE channel_hash = ? AND payload_type = 5 AND first_seen >= ?
|
||
GROUP BY bucket
|
||
ORDER BY bucket
|
||
`, bucketExpr)
|
||
tsRows, err := db.conn.Query(tsQuery, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving timeseries query: %w", err)
|
||
}
|
||
resp.TimeSeries = make([]WardrivingTimePoint, 0)
|
||
for tsRows.Next() {
|
||
var pt WardrivingTimePoint
|
||
if tsRows.Scan(&pt.T, &pt.Count) == nil {
|
||
resp.TimeSeries = append(resp.TimeSeries, pt)
|
||
}
|
||
}
|
||
tsRows.Close()
|
||
if err := tsRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving timeseries iteration: %w", err)
|
||
}
|
||
|
||
senderRows, err := db.conn.Query(`
|
||
SELECT json_extract(decoded_json, '$.sender') AS sender, COUNT(*) AS cnt
|
||
FROM transmissions
|
||
WHERE channel_hash = ? AND payload_type = 5 AND first_seen >= ?
|
||
AND json_extract(decoded_json, '$.sender') IS NOT NULL
|
||
AND json_extract(decoded_json, '$.sender') != ''
|
||
GROUP BY sender
|
||
ORDER BY cnt DESC
|
||
`, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving senders query: %w", err)
|
||
}
|
||
resp.TopSenders = make([]WardrivingSenderCount, 0)
|
||
for senderRows.Next() {
|
||
var sc WardrivingSenderCount
|
||
if senderRows.Scan(&sc.Sender, &sc.Count) == nil {
|
||
resp.TopSenders = append(resp.TopSenders, sc)
|
||
}
|
||
}
|
||
senderRows.Close()
|
||
if err := senderRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving senders iteration: %w", err)
|
||
}
|
||
|
||
entryRows, err := db.conn.Query(`
|
||
SELECT json_extract(o.path_json, '$[0]') AS prefix,
|
||
COUNT(*) AS observation_count,
|
||
COUNT(DISTINCT o.transmission_id) AS message_count
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?
|
||
AND o.path_json IS NOT NULL AND json_array_length(o.path_json) > 0
|
||
GROUP BY prefix
|
||
ORDER BY observation_count DESC
|
||
`, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving entry points query: %w", err)
|
||
}
|
||
resp.EntryPoints = make([]WardrivingEntryPrefix, 0)
|
||
for entryRows.Next() {
|
||
var ep WardrivingEntryPrefix
|
||
if entryRows.Scan(&ep.Prefix, &ep.ObservationCount, &ep.MessageCount) == nil {
|
||
resp.EntryPoints = append(resp.EntryPoints, ep)
|
||
}
|
||
}
|
||
entryRows.Close()
|
||
if err := entryRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving entry points iteration: %w", err)
|
||
}
|
||
|
||
var obsQuery string
|
||
if db.isV3 {
|
||
obsQuery = `
|
||
SELECT obs.rowid AS observer_id, obs.name, COALESCE(obs.iata, ''),
|
||
COUNT(*) AS observation_count, COUNT(DISTINCT o.transmission_id) AS message_count
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?
|
||
GROUP BY observer_id
|
||
ORDER BY observation_count DESC`
|
||
} else {
|
||
obsQuery = `
|
||
SELECT obs.id AS observer_id, obs.name, COALESCE(obs.iata, ''),
|
||
COUNT(*) AS observation_count, COUNT(DISTINCT o.transmission_id) AS message_count
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
JOIN observers obs ON obs.id = o.observer_id
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?
|
||
GROUP BY observer_id
|
||
ORDER BY observation_count DESC`
|
||
}
|
||
obsRows, err := db.conn.Query(obsQuery, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving observers query: %w", err)
|
||
}
|
||
resp.Observers = make([]WardrivingObserverCoverage, 0)
|
||
for obsRows.Next() {
|
||
var oc WardrivingObserverCoverage
|
||
var name sql.NullString
|
||
if err := obsRows.Scan(&oc.ObserverID, &name, &oc.IATA, &oc.ObservationCount, &oc.MessageCount); err != nil {
|
||
continue
|
||
}
|
||
oc.ObserverName = name.String
|
||
if oc.ObserverName == "" {
|
||
oc.ObserverName = oc.ObserverID
|
||
}
|
||
if coord, ok := iataCoords[strings.ToUpper(strings.TrimSpace(oc.IATA))]; ok {
|
||
lat, lon := coord.Lat, coord.Lon
|
||
oc.Lat, oc.Lon = &lat, &lon
|
||
}
|
||
resp.Observers = append(resp.Observers, oc)
|
||
}
|
||
obsRows.Close()
|
||
if err := obsRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving observers iteration: %w", err)
|
||
}
|
||
|
||
// Signal quality over time — same bucketing as the activity time series,
|
||
// but averaged across every observation (any observer) in that bucket.
|
||
sigBucketExpr := strings.ReplaceAll(bucketExpr, "first_seen", "t.first_seen")
|
||
sigQuery := fmt.Sprintf(`
|
||
SELECT %s AS bucket, AVG(o.snr) AS avg_snr, AVG(o.rssi) AS avg_rssi, COUNT(*) AS cnt
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?
|
||
GROUP BY bucket
|
||
ORDER BY bucket
|
||
`, sigBucketExpr)
|
||
sigRows, err := db.conn.Query(sigQuery, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving signal timeseries query: %w", err)
|
||
}
|
||
resp.SignalTimeSeries = make([]WardrivingSignalPoint, 0)
|
||
for sigRows.Next() {
|
||
var sp WardrivingSignalPoint
|
||
if sigRows.Scan(&sp.T, &sp.AvgSNR, &sp.AvgRSSI, &sp.ObservationCount) == nil {
|
||
resp.SignalTimeSeries = append(resp.SignalTimeSeries, sp)
|
||
}
|
||
}
|
||
sigRows.Close()
|
||
if err := sigRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving signal timeseries iteration: %w", err)
|
||
}
|
||
|
||
var avgSNR, avgRSSI sql.NullFloat64
|
||
if err := db.conn.QueryRow(`
|
||
SELECT AVG(o.snr), AVG(o.rssi)
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?
|
||
`, channel, since).Scan(&avgSNR, &avgRSSI); err != nil {
|
||
return nil, fmt.Errorf("wardriving avg signal query: %w", err)
|
||
}
|
||
if avgSNR.Valid {
|
||
v := avgSNR.Float64
|
||
resp.AvgSNR = &v
|
||
}
|
||
if avgRSSI.Valid {
|
||
v := avgRSSI.Float64
|
||
resp.AvgRSSI = &v
|
||
}
|
||
|
||
sessions, err := db.buildWardrivingSessions(channel, since)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
resp.Sessions = sessions
|
||
|
||
gpsShares, err := db.detectWardrivingGPSShares(channel, since)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
resp.GPSShares = gpsShares
|
||
|
||
return resp, nil
|
||
}
|
||
|
||
// wardrivingSessionGapMinutes is the max gap between two consecutive
|
||
// messages from the same sender before buildWardrivingSessions treats them
|
||
// as separate wardriving runs rather than one continuous session.
|
||
const wardrivingSessionGapMinutes = 15.0
|
||
|
||
// buildWardrivingSessions groups each sender's messages (ordered by time)
|
||
// into runs, splitting on any gap over wardrivingSessionGapMinutes. For
|
||
// each session it also computes how many distinct entry-point repeaters
|
||
// and observers were involved, by unioning the per-transmission
|
||
// observation data across every message in that session.
|
||
func (db *DB) buildWardrivingSessions(channel, since string) ([]WardrivingSession, error) {
|
||
msgRows, err := db.conn.Query(`
|
||
SELECT id, json_extract(decoded_json, '$.sender') AS sender, first_seen
|
||
FROM transmissions
|
||
WHERE channel_hash = ? AND payload_type = 5 AND first_seen >= ?
|
||
AND json_extract(decoded_json, '$.sender') IS NOT NULL
|
||
AND json_extract(decoded_json, '$.sender') != ''
|
||
ORDER BY sender, first_seen ASC
|
||
`, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving sessions message query: %w", err)
|
||
}
|
||
type txInfo struct {
|
||
id int64
|
||
sender string
|
||
ts time.Time
|
||
}
|
||
var txs []txInfo
|
||
for msgRows.Next() {
|
||
var id int64
|
||
var sender, tsStr string
|
||
if err := msgRows.Scan(&id, &sender, &tsStr); err != nil {
|
||
continue
|
||
}
|
||
ts, err := time.Parse(time.RFC3339, tsStr)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
txs = append(txs, txInfo{id: id, sender: sender, ts: ts})
|
||
}
|
||
msgRows.Close()
|
||
if err := msgRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving sessions message iteration: %w", err)
|
||
}
|
||
|
||
// Per-transmission entry-point prefixes and observer IDs, so each
|
||
// session can report how many distinct ones it touched.
|
||
var perTxQuery string
|
||
if db.isV3 {
|
||
perTxQuery = `
|
||
SELECT o.transmission_id, json_extract(o.path_json, '$[0]'), obs.rowid
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?`
|
||
} else {
|
||
perTxQuery = `
|
||
SELECT o.transmission_id, json_extract(o.path_json, '$[0]'), obs.id
|
||
FROM observations o
|
||
JOIN transmissions t ON t.id = o.transmission_id
|
||
JOIN observers obs ON obs.id = o.observer_id
|
||
WHERE t.channel_hash = ? AND t.payload_type = 5 AND t.first_seen >= ?`
|
||
}
|
||
perTxRows, err := db.conn.Query(perTxQuery, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving sessions per-tx query: %w", err)
|
||
}
|
||
txPrefixes := make(map[int64]map[string]bool)
|
||
txObservers := make(map[int64]map[string]bool)
|
||
for perTxRows.Next() {
|
||
var txID int64
|
||
var prefix sql.NullString
|
||
var observerID string
|
||
if err := perTxRows.Scan(&txID, &prefix, &observerID); err != nil {
|
||
continue
|
||
}
|
||
if prefix.Valid && prefix.String != "" {
|
||
if txPrefixes[txID] == nil {
|
||
txPrefixes[txID] = make(map[string]bool)
|
||
}
|
||
txPrefixes[txID][prefix.String] = true
|
||
}
|
||
if txObservers[txID] == nil {
|
||
txObservers[txID] = make(map[string]bool)
|
||
}
|
||
txObservers[txID][observerID] = true
|
||
}
|
||
perTxRows.Close()
|
||
if err := perTxRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving sessions per-tx iteration: %w", err)
|
||
}
|
||
|
||
sessions := make([]WardrivingSession, 0)
|
||
var cur *WardrivingSession
|
||
var curPrefixes, curObservers map[string]bool
|
||
var lastTS time.Time
|
||
flush := func() {
|
||
if cur == nil {
|
||
return
|
||
}
|
||
cur.EntryPointCount = len(curPrefixes)
|
||
cur.ObserverCount = len(curObservers)
|
||
for p := range curPrefixes {
|
||
cur.EntryPointPrefixes = append(cur.EntryPointPrefixes, p)
|
||
}
|
||
sort.Strings(cur.EntryPointPrefixes)
|
||
start, errS := time.Parse(time.RFC3339, cur.StartTime)
|
||
end, errE := time.Parse(time.RFC3339, cur.EndTime)
|
||
if errS == nil && errE == nil {
|
||
cur.DurationMinutes = end.Sub(start).Minutes()
|
||
}
|
||
sessions = append(sessions, *cur)
|
||
}
|
||
for _, tx := range txs {
|
||
newSession := cur == nil || cur.Sender != tx.sender || tx.ts.Sub(lastTS).Minutes() > wardrivingSessionGapMinutes
|
||
if newSession {
|
||
flush()
|
||
cur = &WardrivingSession{Sender: tx.sender, StartTime: tx.ts.UTC().Format(time.RFC3339)}
|
||
curPrefixes = make(map[string]bool)
|
||
curObservers = make(map[string]bool)
|
||
}
|
||
cur.EndTime = tx.ts.UTC().Format(time.RFC3339)
|
||
cur.MessageCount++
|
||
cur.TransmissionIDs = append(cur.TransmissionIDs, tx.id)
|
||
for p := range txPrefixes[tx.id] {
|
||
curPrefixes[p] = true
|
||
}
|
||
for o := range txObservers[tx.id] {
|
||
curObservers[o] = true
|
||
}
|
||
lastTS = tx.ts
|
||
}
|
||
flush()
|
||
|
||
sort.Slice(sessions, func(i, j int) bool { return sessions[i].StartTime > sessions[j].StartTime })
|
||
return sessions, nil
|
||
}
|
||
|
||
// wardrivingGPSSharePattern matches the plaintext coordinate suffix some
|
||
// wardriving clients append after the standard token — e.g.
|
||
// "MM:c3e_zJ1rUA:55.59743,13.00128" — confirmed empirically against live
|
||
// traffic. This is an explicit choice by that sender's client to share
|
||
// their position in-band; CoreScope reads the plaintext numbers, it does
|
||
// not decode or infer them from the token itself.
|
||
var wardrivingGPSSharePattern = regexp.MustCompile(`^[A-Za-z0-9_-]+:(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)$`)
|
||
|
||
// detectWardrivingGPSShares scans every #wardriving message — stored as
|
||
// "<sender>: MM:<message>" in decoded_json.text, per decodeGrpTxt's
|
||
// "<sender>: <message>" convention — for that coordinate-suffix shape, and
|
||
// returns one entry per sender with the most recent position they shared.
|
||
// Messages without a coordinate suffix (the standard anonymous token, or
|
||
// anything else) are ignored entirely.
|
||
func (db *DB) detectWardrivingGPSShares(channel, since string) ([]WardrivingGPSShare, error) {
|
||
rows, err := db.conn.Query(`
|
||
SELECT json_extract(decoded_json, '$.sender'), json_extract(decoded_json, '$.text'), first_seen
|
||
FROM transmissions
|
||
WHERE channel_hash = ? AND payload_type = 5 AND first_seen >= ?
|
||
ORDER BY first_seen ASC
|
||
`, channel, since)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving gps-share query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
type shareAgg struct {
|
||
lat, lon float64
|
||
count int
|
||
lastSeen string
|
||
}
|
||
agg := make(map[string]*shareAgg)
|
||
|
||
for rows.Next() {
|
||
var sender, text sql.NullString
|
||
var ts string
|
||
if err := rows.Scan(&sender, &text, &ts); err != nil {
|
||
continue
|
||
}
|
||
if !sender.Valid || !text.Valid || sender.String == "" {
|
||
continue
|
||
}
|
||
// decodeGrpTxt (cmd/ingestor/decoder.go) builds Text as
|
||
// "<sender>: <message>", so the wardriving payload prefix is
|
||
// "<sender>: MM:" — not a bare "MM:" at the start of text.
|
||
prefix := sender.String + ": MM:"
|
||
if !strings.HasPrefix(text.String, prefix) {
|
||
continue
|
||
}
|
||
rest := strings.TrimPrefix(text.String, prefix)
|
||
m := wardrivingGPSSharePattern.FindStringSubmatch(rest)
|
||
if m == nil {
|
||
continue
|
||
}
|
||
lat, errLat := strconv.ParseFloat(m[1], 64)
|
||
lon, errLon := strconv.ParseFloat(m[2], 64)
|
||
if errLat != nil || errLon != nil || lat < -90 || lat > 90 || lon < -180 || lon > 180 {
|
||
continue
|
||
}
|
||
a := agg[sender.String]
|
||
if a == nil {
|
||
a = &shareAgg{}
|
||
agg[sender.String] = a
|
||
}
|
||
a.count++
|
||
// Rows are ordered first_seen ASC, so the last write below wins
|
||
// as the most recent shared position.
|
||
a.lat, a.lon, a.lastSeen = lat, lon, ts
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving gps-share iteration: %w", err)
|
||
}
|
||
|
||
shares := make([]WardrivingGPSShare, 0, len(agg))
|
||
for senderName, a := range agg {
|
||
shares = append(shares, WardrivingGPSShare{
|
||
Sender: senderName,
|
||
Lat: a.lat,
|
||
Lon: a.lon,
|
||
MessageCount: a.count,
|
||
LastSeen: a.lastSeen,
|
||
})
|
||
}
|
||
sort.Slice(shares, func(i, j int) bool { return shares[i].LastSeen > shares[j].LastSeen })
|
||
return shares, nil
|
||
}
|
||
|
||
// wardrivingSenderMessagesLimit caps how many individual messages
|
||
// GetWardrivingSenderMessages returns per request — this is a drill-down
|
||
// view (one sender, optionally one session), not a bulk export.
|
||
const wardrivingSenderMessagesLimit = 200
|
||
|
||
// GetWardrivingSenderMessages returns one sender's individual #wardriving
|
||
// messages in [since, until], most-recent-first: each message's entry-point
|
||
// path (path[0] first, same convention as WardrivingEntryPrefix), the
|
||
// observers that heard it with their own SNR/RSSI, and Lat/Lon when that
|
||
// specific message carried an explicit shared position (see
|
||
// detectWardrivingGPSShares). This is the per-message detail behind the
|
||
// aggregate Sessions/Entry Points/Coverage views — used when a user drills
|
||
// into one sender or one session.
|
||
func (db *DB) GetWardrivingSenderMessages(sender, channel, since, until string) (*WardrivingSenderMessagesResponse, error) {
|
||
resp := &WardrivingSenderMessagesResponse{
|
||
Sender: sender, Channel: channel, Since: since, Until: until,
|
||
Messages: make([]WardrivingMessage, 0),
|
||
}
|
||
|
||
txRows, err := db.conn.Query(`
|
||
SELECT id, first_seen, json_extract(decoded_json, '$.text')
|
||
FROM transmissions
|
||
WHERE channel_hash = ? AND payload_type = 5 AND first_seen >= ? AND first_seen <= ?
|
||
AND json_extract(decoded_json, '$.sender') = ?
|
||
ORDER BY first_seen DESC
|
||
LIMIT ?
|
||
`, channel, since, until, sender, wardrivingSenderMessagesLimit)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving sender messages query: %w", err)
|
||
}
|
||
type txRow struct {
|
||
id int64
|
||
ts string
|
||
text sql.NullString
|
||
}
|
||
var txs []txRow
|
||
for txRows.Next() {
|
||
var t txRow
|
||
if err := txRows.Scan(&t.id, &t.ts, &t.text); err != nil {
|
||
continue
|
||
}
|
||
txs = append(txs, t)
|
||
}
|
||
txRows.Close()
|
||
if err := txRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving sender messages iteration: %w", err)
|
||
}
|
||
if len(txs) == 0 {
|
||
return resp, nil
|
||
}
|
||
|
||
placeholders := make([]string, len(txs))
|
||
args := make([]interface{}, len(txs))
|
||
txIndex := make(map[int64]int, len(txs))
|
||
for i, t := range txs {
|
||
placeholders[i] = "?"
|
||
args[i] = t.id
|
||
txIndex[t.id] = i
|
||
}
|
||
|
||
var obsQuery string
|
||
if db.isV3 {
|
||
obsQuery = fmt.Sprintf(`
|
||
SELECT o.transmission_id, obs.name, o.snr, o.rssi, o.path_json
|
||
FROM observations o
|
||
JOIN observers obs ON obs.rowid = o.observer_idx
|
||
WHERE o.transmission_id IN (%s)`, strings.Join(placeholders, ","))
|
||
} else {
|
||
obsQuery = fmt.Sprintf(`
|
||
SELECT o.transmission_id, obs.name, o.snr, o.rssi, o.path_json
|
||
FROM observations o
|
||
JOIN observers obs ON obs.id = o.observer_id
|
||
WHERE o.transmission_id IN (%s)`, strings.Join(placeholders, ","))
|
||
}
|
||
obsRows, err := db.conn.Query(obsQuery, args...)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("wardriving sender messages observations query: %w", err)
|
||
}
|
||
|
||
type msgAgg struct {
|
||
observations []WardrivingMessageObservation
|
||
pathPrefixes []string
|
||
}
|
||
aggs := make([]*msgAgg, len(txs))
|
||
for i := range aggs {
|
||
aggs[i] = &msgAgg{observations: make([]WardrivingMessageObservation, 0), pathPrefixes: make([]string, 0)}
|
||
}
|
||
|
||
for obsRows.Next() {
|
||
var txID int64
|
||
var observerName sql.NullString
|
||
var snr, rssi sql.NullFloat64
|
||
var pathJSON sql.NullString
|
||
if err := obsRows.Scan(&txID, &observerName, &snr, &rssi, &pathJSON); err != nil {
|
||
continue
|
||
}
|
||
idx, ok := txIndex[txID]
|
||
if !ok {
|
||
continue
|
||
}
|
||
agg := aggs[idx]
|
||
agg.observations = append(agg.observations, WardrivingMessageObservation{
|
||
ObserverName: observerName.String,
|
||
SNR: snr.Float64,
|
||
RSSI: rssi.Float64,
|
||
})
|
||
if pathJSON.Valid && pathJSON.String != "" {
|
||
var path []string
|
||
if json.Unmarshal([]byte(pathJSON.String), &path) == nil && len(path) > len(agg.pathPrefixes) {
|
||
agg.pathPrefixes = path
|
||
}
|
||
}
|
||
}
|
||
obsRows.Close()
|
||
if err := obsRows.Err(); err != nil {
|
||
return nil, fmt.Errorf("wardriving sender messages observations iteration: %w", err)
|
||
}
|
||
|
||
for i, t := range txs {
|
||
msg := WardrivingMessage{
|
||
TransmissionID: t.id,
|
||
Timestamp: t.ts,
|
||
PathPrefixes: aggs[i].pathPrefixes,
|
||
Observations: aggs[i].observations,
|
||
}
|
||
if t.text.Valid {
|
||
prefix := sender + ": MM:"
|
||
if strings.HasPrefix(t.text.String, prefix) {
|
||
rest := strings.TrimPrefix(t.text.String, prefix)
|
||
if m := wardrivingGPSSharePattern.FindStringSubmatch(rest); m != nil {
|
||
lat, errLat := strconv.ParseFloat(m[1], 64)
|
||
lon, errLon := strconv.ParseFloat(m[2], 64)
|
||
if errLat == nil && errLon == nil && lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180 {
|
||
msg.Lat = &lat
|
||
msg.Lon = &lon
|
||
}
|
||
}
|
||
}
|
||
}
|
||
resp.Messages = append(resp.Messages, msg)
|
||
}
|
||
|
||
return resp, nil
|
||
}
|
||
|
||
// GetMatchedRegionNames returns the set of scope_name values that have ever
|
||
// matched at least one transmission still in retention (NULL and empty-string
|
||
// "unknown" rows are excluded). Used to diff against the operator's
|
||
// configured hashRegions list and surface which configured regions have
|
||
// never actually matched anything — region-utilization analytics.
|
||
func (db *DB) GetMatchedRegionNames() (map[string]bool, error) {
|
||
matched := make(map[string]bool)
|
||
if !db.hasScopeName {
|
||
return matched, nil
|
||
}
|
||
rows, err := db.conn.Query(`SELECT DISTINCT scope_name FROM transmissions WHERE scope_name IS NOT NULL AND scope_name != ''`)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("matched region names query: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
for rows.Next() {
|
||
var name string
|
||
if rows.Scan(&name) == nil {
|
||
matched[name] = true
|
||
}
|
||
}
|
||
return matched, rows.Err()
|
||
}
|
||
|
||
// NodeForGeoPrune holds the minimal fields needed for geo-filter pruning.
|
||
type NodeForGeoPrune struct {
|
||
PubKey string
|
||
Name string
|
||
Lat *float64
|
||
Lon *float64
|
||
}
|
||
|
||
// GetNodesForGeoPrune returns all nodes with their coordinates for geo-filter evaluation.
|
||
// Read-only — safe on the server's mode=ro handle.
|
||
func (db *DB) GetNodesForGeoPrune() ([]NodeForGeoPrune, error) {
|
||
rows, err := db.conn.Query("SELECT public_key, name, lat, lon FROM nodes ORDER BY name")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
|
||
var nodes []NodeForGeoPrune
|
||
for rows.Next() {
|
||
var pk string
|
||
var name sql.NullString
|
||
var lat, lon sql.NullFloat64
|
||
if err := rows.Scan(&pk, &name, &lat, &lon); err != nil {
|
||
continue
|
||
}
|
||
n := NodeForGeoPrune{PubKey: pk, Name: name.String}
|
||
if lat.Valid {
|
||
v := lat.Float64
|
||
n.Lat = &v
|
||
}
|
||
if lon.Valid {
|
||
v := lon.Float64
|
||
n.Lon = &v
|
||
}
|
||
nodes = append(nodes, n)
|
||
}
|
||
return nodes, rows.Err()
|
||
}
|
||
|
||
// DeleteNodesByPubkeys was removed in PR #738 follow-up: server is read-only
|
||
// (opened with mode=ro after #1283/#1289), so DELETE statements would fail at
|
||
// runtime. Geo-prune now flows server → marker file → ingestor; see
|
||
// internal/prunequeue and cmd/ingestor/prune_geofilter.go.
|