From 2435f2eaaf5af5e209651a28993df4aee9f19ddc Mon Sep 17 00:00:00 2001 From: Kpa-clawbot <259247574+Kpa-clawbot@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:09:36 -0700 Subject: [PATCH] fix: observation timestamps, leaked fields, perf path normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #178: Use strftime ISO 8601 format instead of datetime() for observation timestamps in all SQL queries (v3 + v2 views). Add normalizeTimestamp() helper for non-v3 paths that may store space-separated timestamps. - #179: Strip internal fields (decoded_json, direction, payload_type, raw_hex, route_type, score, created_at) from ObservationResp. Only expose id, transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp — matching Node.js parity. - #180: Remove _parsedDecoded and _parsedPath from node detail recentAdverts response. These internal/computed fields were leaking to the API. Updated golden shapes.json accordingly. - #181: Use mux route template (GetPathTemplate) for perf stats path normalization, converting {param} to :param for Node.js parity. Fallback to hex regex for unmatched routes. Compile regexes once at package level instead of per-request. fixes #178, fixes #179, fixes #180, fixes #181 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cmd/server/coverage_test.go | 2 +- cmd/server/db.go | 40 ++++++-------------------- cmd/server/db_test.go | 2 +- cmd/server/routes.go | 26 ++++++++++------- cmd/server/store.go | 24 ++++++++++++---- cmd/server/testdata/golden/shapes.json | 38 ------------------------ cmd/server/types.go | 7 ----- 7 files changed, 44 insertions(+), 95 deletions(-) diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 238ec91..5bd6a42 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -47,7 +47,7 @@ func setupTestDBv2(t *testing.T) *DB { ); CREATE VIEW packets_v AS SELECT o.id, t.raw_hex, - datetime(o.timestamp, 'unixepoch') AS timestamp, + strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, o.observer_id, o.observer_name, o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, t.payload_type, t.payload_version, o.path_json, t.decoded_json, t.created_at diff --git a/cmd/server/db.go b/cmd/server/db.go index 3f38b21..9757628 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -443,7 +443,7 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(datetime(oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, + COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, obs.id AS observer_id, obs.name AS observer_name, o.snr, o.rssi, o.path_json FROM transmissions t @@ -850,34 +850,6 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, name string, limit in for rows.Next() { p := db.scanTransmissionRow(rows) if p != nil { - // Parse _parsedPath from path_json - if pj, ok := p["path_json"].(string); ok && pj != "" { - var pathArr []interface{} - if json.Unmarshal([]byte(pj), &pathArr) == nil { - strs := make([]string, 0, len(pathArr)) - for _, v := range pathArr { - if s, ok := v.(string); ok { - strs = append(strs, s) - } - } - p["_parsedPath"] = strs - } else { - p["_parsedPath"] = []string{} - } - } else { - p["_parsedPath"] = []string{} - } - // Parse _parsedDecoded from decoded_json - if dj, ok := p["decoded_json"].(string); ok && dj != "" { - var decoded map[string]interface{} - if json.Unmarshal([]byte(dj), &decoded) == nil { - p["_parsedDecoded"] = decoded - } else { - p["_parsedDecoded"] = map[string]interface{}{} - } - } else { - p["_parsedDecoded"] = map[string]interface{}{} - } // Placeholder for observations — filled below p["observations"] = []map[string]interface{}{} if id, ok := p["id"].(int); ok { @@ -921,7 +893,7 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string] var querySQL string if db.isV3 { querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, - o.direction, o.snr, o.rssi, o.path_json, datetime(o.timestamp, 'unixepoch') AS obs_timestamp + o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE o.transmission_id IN (%s) @@ -950,16 +922,20 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string] continue } + ts := nullStr(obsTimestamp) + if s, ok := ts.(string); ok { + ts = normalizeTimestamp(s) + } + obs := map[string]interface{}{ "id": obsID, "transmission_id": txID, "observer_id": nullStr(observerID), "observer_name": nullStr(observerName), - "direction": nullStr(direction), "snr": nullFloat(snr), "rssi": nullFloat(rssi), "path_json": nullStr(pathJSON), - "timestamp": nullStr(obsTimestamp), + "timestamp": ts, } result[txID] = append(result[txID], obs) } diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 385d032..d661bca 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -73,7 +73,7 @@ func setupTestDB(t *testing.T) *DB { CREATE VIEW packets_v AS SELECT o.id, t.raw_hex, - datetime(o.timestamp, 'unixepoch') AS timestamp, + strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, obs.id AS observer_id, obs.name AS observer_name, o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, t.payload_type, t.payload_version, o.path_json, t.decoded_json, diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 1d7e1be..f063381 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -137,9 +137,16 @@ func (s *Server) perfMiddleware(next http.Handler) http.Handler { s.perfStats.Requests++ s.perfStats.TotalMs += ms - // Normalize key - re := regexp.MustCompile(`[0-9a-f]{8,}`) - key := re.ReplaceAllString(r.URL.Path, ":id") + // Normalize key: prefer mux route template (like Node.js req.route.path) + key := r.URL.Path + if route := mux.CurrentRoute(r); route != nil { + if tmpl, err := route.GetPathTemplate(); err == nil { + key = muxBraceParam.ReplaceAllString(tmpl, ":$1") + } + } + if key == r.URL.Path { + key = perfHexFallback.ReplaceAllString(key, ":id") + } if _, ok := s.perfStats.Endpoints[key]; !ok { s.perfStats.Endpoints[key] = &EndpointPerf{Recent: make([]float64, 0, 100)} } @@ -595,6 +602,12 @@ func (s *Server) handlePacketTimestamps(w http.ResponseWriter, r *http.Request) var hashPattern = regexp.MustCompile(`^[0-9a-f]{16}$`) +// muxBraceParam matches {param} in gorilla/mux route templates for normalization. +var muxBraceParam = regexp.MustCompile(`\{([^}]+)\}`) + +// perfHexFallback matches hex IDs for perf path normalization fallback. +var perfHexFallback = regexp.MustCompile(`[0-9a-f]{8,}`) + func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) { param := mux.Vars(r)["id"] var packet map[string]interface{} @@ -2215,17 +2228,10 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp { obs.Hash = m["hash"] obs.ObserverID = m["observer_id"] obs.ObserverName = m["observer_name"] - obs.Direction = m["direction"] obs.SNR = m["snr"] obs.RSSI = m["rssi"] - obs.Score = m["score"] obs.PathJSON = m["path_json"] obs.Timestamp = m["timestamp"] - obs.RawHex = m["raw_hex"] - obs.PayloadType = m["payload_type"] - obs.DecodedJSON = m["decoded_json"] - obs.RouteType = m["route_type"] - obs.CreatedAt = m["created_at"] result = append(result, obs) } return result diff --git a/cmd/server/store.go b/cmd/server/store.go index 2a108aa..13a84ed 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -129,7 +129,7 @@ func (s *PacketStore) Load() error { loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch') + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx @@ -216,7 +216,7 @@ func (s *PacketStore) Load() error { RSSI: nullFloatPtr(rssi), Score: nullIntPtr(score), PathJSON: obsPJ, - Timestamp: nullStrVal(obsTimestamp), + Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), } tx.Observations = append(tx.Observations, obs) @@ -758,7 +758,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch') + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx @@ -913,7 +913,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac RSSI: r.rssi, Score: r.score, PathJSON: r.pathJSON, - Timestamp: r.obsTS, + Timestamp: normalizeTimestamp(r.obsTS), } tx.Observations = append(tx.Observations, obs) tx.ObservationCount++ @@ -1002,7 +1002,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) int { var querySQL string if s.db.isV3 { querySQL = `SELECT o.id, o.transmission_id, obs.id, obs.name, o.direction, - o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch') + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE o.id > ? @@ -1109,7 +1109,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) int { RSSI: r.rssi, Score: r.score, PathJSON: r.pathJSON, - Timestamp: r.timestamp, + Timestamp: normalizeTimestamp(r.timestamp), } tx.Observations = append(tx.Observations, obs) tx.ObservationCount++ @@ -1345,6 +1345,18 @@ func strOrNil(s string) interface{} { return s } +// normalizeTimestamp converts SQLite datetime format ("YYYY-MM-DD HH:MM:SS") +// to ISO 8601 ("YYYY-MM-DDTHH:MM:SSZ"). Already-ISO strings pass through. +func normalizeTimestamp(s string) string { + if s == "" { + return s + } + if t, err := time.Parse("2006-01-02 15:04:05", s); err == nil { + return t.UTC().Format("2006-01-02T15:04:05.000Z") + } + return s +} + func intPtrOrNil(p *int) interface{} { if p == nil { return nil diff --git a/cmd/server/testdata/golden/shapes.json b/cmd/server/testdata/golden/shapes.json index d5267dc..b33fb71 100644 --- a/cmd/server/testdata/golden/shapes.json +++ b/cmd/server/testdata/golden/shapes.json @@ -1046,44 +1046,6 @@ }, "path_json": { "type": "string" - }, - "_parsedPath": { - "type": "array", - "elementShape": { - "type": "string" - } - }, - "_parsedDecoded": { - "type": "object", - "keys": { - "type": { - "type": "string" - }, - "pubKey": { - "type": "string" - }, - "timestamp": { - "type": "number" - }, - "timestampISO": { - "type": "string" - }, - "signature": { - "type": "string" - }, - "flags": { - "type": "object" - }, - "lat": { - "type": "number" - }, - "lon": { - "type": "number" - }, - "name": { - "type": "string" - } - } } } } diff --git a/cmd/server/types.go b/cmd/server/types.go index 0c10113..6a8978a 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -236,17 +236,10 @@ type ObservationResp struct { Hash interface{} `json:"hash,omitempty"` ObserverID interface{} `json:"observer_id"` ObserverName interface{} `json:"observer_name"` - Direction interface{} `json:"direction"` SNR interface{} `json:"snr"` RSSI interface{} `json:"rssi"` - Score interface{} `json:"score"` PathJSON interface{} `json:"path_json"` Timestamp interface{} `json:"timestamp"` - RawHex interface{} `json:"raw_hex,omitempty"` - PayloadType interface{} `json:"payload_type,omitempty"` - DecodedJSON interface{} `json:"decoded_json,omitempty"` - RouteType interface{} `json:"route_type,omitempty"` - CreatedAt interface{} `json:"created_at,omitempty"` } type GroupedPacketResp struct {