dandri/meshcore-analyzer
1
0
Fork 0
mirror of https://github.com/Kpa-clawbot/meshcore-analyzer.git synced 2026-03-30 13:35:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: observation timestamps, leaked fields, perf path normalization

Browse Source 
- #178: Use strftime ISO 8601 format instead of datetime() for observation
  timestamps in all SQL queries (v3 + v2 views). Add normalizeTimestamp()
  helper for non-v3 paths that may store space-separated timestamps.

- #179: Strip internal fields (decoded_json, direction, payload_type,
  raw_hex, route_type, score, created_at) from ObservationResp. Only
  expose id, transmission_id, observer_id, observer_name, snr, rssi,
  path_json, timestamp — matching Node.js parity.

- #180: Remove _parsedDecoded and _parsedPath from node detail
  recentAdverts response. These internal/computed fields were leaking
  to the API. Updated golden shapes.json accordingly.

- #181: Use mux route template (GetPathTemplate) for perf stats path
  normalization, converting {param} to :param for Node.js parity.
  Fallback to hex regex for unmatched routes. Compile regexes once at
  package level instead of per-request.

fixes #178, fixes #179, fixes #180, fixes #181

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Kpa-clawbot
2026-03-27 18:09:36 -07:00
parent 2c9c6503fb
commit 2435f2eaaf
7 changed files with 44 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/server/coverage_test.go
View File

@@ -47,7 +47,7 @@ func setupTestDBv2(t *testing.T) *DB {
);
CREATE VIEW packets_v AS
SELECT o.id, t.raw_hex,
datetime(o.timestamp, 'unixepoch') AS timestamp,
strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp,
o.observer_id, o.observer_name,
o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type,
t.payload_type, t.payload_version, o.path_json, t.decoded_json, t.created_at

40
cmd/server/db.go
View File

@@ -443,7 +443,7 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) {
querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type,
COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count,
COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count,
COALESCE((SELECT MAX(datetime(oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest,
COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest,
obs.id AS observer_id, obs.name AS observer_name,
o.snr, o.rssi, o.path_json
FROM transmissions t
@@ -850,34 +850,6 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, name string, limit in
for rows.Next() {
p := db.scanTransmissionRow(rows)
if p != nil {
// Parse _parsedPath from path_json
if pj, ok := p["path_json"].(string); ok && pj != "" {
var pathArr []interface{}
if json.Unmarshal([]byte(pj), &pathArr) == nil {
strs := make([]string, 0, len(pathArr))
for _, v := range pathArr {
if s, ok := v.(string); ok {
strs = append(strs, s)
}
}
p["_parsedPath"] = strs
} else {
p["_parsedPath"] = []string{}
}
} else {
p["_parsedPath"] = []string{}
}
// Parse _parsedDecoded from decoded_json
if dj, ok := p["decoded_json"].(string); ok && dj != "" {
var decoded map[string]interface{}
if json.Unmarshal([]byte(dj), &decoded) == nil {
p["_parsedDecoded"] = decoded
} else {
p["_parsedDecoded"] = map[string]interface{}{}
}
} else {
p["_parsedDecoded"] = map[string]interface{}{}
}
// Placeholder for observations — filled below
p["observations"] = []map[string]interface{}{}
if id, ok := p["id"].(int); ok {
@@ -921,7 +893,7 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string]
var querySQL string
if db.isV3 {
querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name,
o.direction, o.snr, o.rssi, o.path_json, datetime(o.timestamp, 'unixepoch') AS obs_timestamp
o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp
FROM observations o
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
WHERE o.transmission_id IN (%s)
@@ -950,16 +922,20 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string]
continue
}
ts := nullStr(obsTimestamp)
if s, ok := ts.(string); ok {
ts = normalizeTimestamp(s)
}
obs := map[string]interface{}{
"id": obsID,
"transmission_id": txID,
"observer_id": nullStr(observerID),
"observer_name": nullStr(observerName),
"direction": nullStr(direction),
"snr": nullFloat(snr),
"rssi": nullFloat(rssi),
"path_json": nullStr(pathJSON),
"timestamp": nullStr(obsTimestamp),
"timestamp": ts,
}
result[txID] = append(result[txID], obs)
}

2
cmd/server/db_test.go
View File

@@ -73,7 +73,7 @@ func setupTestDB(t *testing.T) *DB {
CREATE VIEW packets_v AS
SELECT o.id, t.raw_hex,
datetime(o.timestamp, 'unixepoch') AS timestamp,
strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp,
obs.id AS observer_id, obs.name AS observer_name,
o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type,
t.payload_type, t.payload_version, o.path_json, t.decoded_json,

26
cmd/server/routes.go
View File

@@ -137,9 +137,16 @@ func (s *Server) perfMiddleware(next http.Handler) http.Handler {
s.perfStats.Requests++
s.perfStats.TotalMs += ms
// Normalize key
re := regexp.MustCompile(`[0-9a-f]{8,}`)
key := re.ReplaceAllString(r.URL.Path, ":id")
// Normalize key: prefer mux route template (like Node.js req.route.path)
key := r.URL.Path
if route := mux.CurrentRoute(r); route != nil {
if tmpl, err := route.GetPathTemplate(); err == nil {
key = muxBraceParam.ReplaceAllString(tmpl, ":$1")
}
}
if key == r.URL.Path {
key = perfHexFallback.ReplaceAllString(key, ":id")
}
if _, ok := s.perfStats.Endpoints[key]; !ok {
s.perfStats.Endpoints[key] = &EndpointPerf{Recent: make([]float64, 0, 100)}
}
@@ -595,6 +602,12 @@ func (s *Server) handlePacketTimestamps(w http.ResponseWriter, r *http.Request)
var hashPattern = regexp.MustCompile(`^[0-9a-f]{16}$`)
// muxBraceParam matches {param} in gorilla/mux route templates for normalization.
var muxBraceParam = regexp.MustCompile(`\{([^}]+)\}`)
// perfHexFallback matches hex IDs for perf path normalization fallback.
var perfHexFallback = regexp.MustCompile(`[0-9a-f]{8,}`)
func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
param := mux.Vars(r)["id"]
var packet map[string]interface{}
@@ -2215,17 +2228,10 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp {
obs.Hash = m["hash"]
obs.ObserverID = m["observer_id"]
obs.ObserverName = m["observer_name"]
obs.Direction = m["direction"]
obs.SNR = m["snr"]
obs.RSSI = m["rssi"]
obs.Score = m["score"]
obs.PathJSON = m["path_json"]
obs.Timestamp = m["timestamp"]
obs.RawHex = m["raw_hex"]
obs.PayloadType = m["payload_type"]
obs.DecodedJSON = m["decoded_json"]
obs.RouteType = m["route_type"]
obs.CreatedAt = m["created_at"]
result = append(result, obs)
}
return result

24
cmd/server/store.go
View File

@@ -129,7 +129,7 @@ func (s *PacketStore) Load() error {
loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
t.payload_type, t.payload_version, t.decoded_json,
o.id, obs.id, obs.name, o.direction,
o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch')
o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')
FROM transmissions t
LEFT JOIN observations o ON o.transmission_id = t.id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
@@ -216,7 +216,7 @@ func (s *PacketStore) Load() error {
RSSI: nullFloatPtr(rssi),
Score: nullIntPtr(score),
PathJSON: obsPJ,
Timestamp: nullStrVal(obsTimestamp),
Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)),
}
tx.Observations = append(tx.Observations, obs)
@@ -758,7 +758,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
t.payload_type, t.payload_version, t.decoded_json,
o.id, obs.id, obs.name, o.direction,
o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch')
o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')
FROM transmissions t
LEFT JOIN observations o ON o.transmission_id = t.id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
@@ -913,7 +913,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac
RSSI: r.rssi,
Score: r.score,
PathJSON: r.pathJSON,
Timestamp: r.obsTS,
Timestamp: normalizeTimestamp(r.obsTS),
}
tx.Observations = append(tx.Observations, obs)
tx.ObservationCount++
@@ -1002,7 +1002,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) int {
var querySQL string
if s.db.isV3 {
querySQL = `SELECT o.id, o.transmission_id, obs.id, obs.name, o.direction,
o.snr, o.rssi, o.score, o.path_json, datetime(o.timestamp, 'unixepoch')
o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')
FROM observations o
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
WHERE o.id > ?
@@ -1109,7 +1109,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) int {
RSSI: r.rssi,
Score: r.score,
PathJSON: r.pathJSON,
Timestamp: r.timestamp,
Timestamp: normalizeTimestamp(r.timestamp),
}
tx.Observations = append(tx.Observations, obs)
tx.ObservationCount++
@@ -1345,6 +1345,18 @@ func strOrNil(s string) interface{} {
return s
}
// normalizeTimestamp converts SQLite datetime format ("YYYY-MM-DD HH:MM:SS")
// to ISO 8601 ("YYYY-MM-DDTHH:MM:SSZ"). Already-ISO strings pass through.
func normalizeTimestamp(s string) string {
if s == "" {
return s
}
if t, err := time.Parse("2006-01-02 15:04:05", s); err == nil {
return t.UTC().Format("2006-01-02T15:04:05.000Z")
}
return s
}
func intPtrOrNil(p *int) interface{} {
if p == nil {
return nil

38
cmd/server/testdata/golden/shapes.json vendored
View File

@@ -1046,44 +1046,6 @@
},
"path_json": {
"type": "string"
},
"_parsedPath": {
"type": "array",
"elementShape": {
"type": "string"
}
},
"_parsedDecoded": {
"type": "object",
"keys": {
"type": {
"type": "string"
},
"pubKey": {
"type": "string"
},
"timestamp": {
"type": "number"
},
"timestampISO": {
"type": "string"
},
"signature": {
"type": "string"
},
"flags": {
"type": "object"
},
"lat": {
"type": "number"
},
"lon": {
"type": "number"
},
"name": {
"type": "string"
}
}
}
}
}

7
cmd/server/types.go
View File

@@ -236,17 +236,10 @@ type ObservationResp struct {
Hash interface{} `json:"hash,omitempty"`
ObserverID interface{} `json:"observer_id"`
ObserverName interface{} `json:"observer_name"`
Direction interface{} `json:"direction"`
SNR interface{} `json:"snr"`
RSSI interface{} `json:"rssi"`
Score interface{} `json:"score"`
PathJSON interface{} `json:"path_json"`
Timestamp interface{} `json:"timestamp"`
RawHex interface{} `json:"raw_hex,omitempty"`
PayloadType interface{} `json:"payload_type,omitempty"`
DecodedJSON interface{} `json:"decoded_json,omitempty"`
RouteType interface{} `json:"route_type,omitempty"`
CreatedAt interface{} `json:"created_at,omitempty"`
}
type GroupedPacketResp struct {