mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-11-11 23:38:17 +00:00
Part 2 of #2260. **Builds on #2261** (certain liveness): review/merge that first; until then this diff also shows its commit. ## Adaptive offline threshold `userManagement.notifications.adaptiveOffline` — **off by default**: ```json "notifications": { "enabled": true, "adaptiveOffline": { "enabled": true, "floorMinutes": 30, "minObservers": 3 } } ``` The fixed `infraSilentHours` (72 h) treats every repeater the same. With this on, a watched node's `node.offline` window is learned from its own **certain** traffic (the #2261 rule): 1. Certain packets per hour over the week **before its last certain packet**. Ending there, not at now, keeps an outage from dragging the rate down until the fixed window returns and the node reads "back online". 2. Empty runs are classified: ≤ 6 h and recurring on another day at the same hours = normal silence (kept; the window is never shorter than the longest + 1 h); any other run = outage, left out of the learning so it neither stretches the window nor blocks learning. 3. λ = 10th percentile of the remaining hourly counts, so quiet nights don't alert. 4. Window = 14/λ hours: silence at rate λ has a < 10⁻⁶ chance (Poisson, ln 10⁶ ≈ 13.8). 5. Clamped to [`floorMinutes`, the role's silent window]. Kept on the fixed window: λ < 1/hour, under 24 h of history in memory, or a 10th-percentile hour heard by fewer than `minObservers` observers (one observer dropping out must not read as the node going offline). Silence is measured up to the newest packet in the store, so a stalling feed stops every node's clock. A node stored offline comes back only on evidence from the moment it went offline on. The notifier caches each learned window for 10 minutes and drops unwatched nodes. For the #2260 repeater (~840 packets/day) the mail would have gone out about 30 minutes into the outage, instead of never (or 72 h with #2261 alone). ## Certain node analytics - Each `activityTimeline` bucket carries `certain`: the part that can only be this node (`certainFor`, the packet-level form of the `certainHeard` rule). `TimeBucket.Certain` is `omitempty`, so other endpoints are unchanged. - Availability and longest silence use certain buckets only; the longest silence now includes one **still running** (an outage in progress is the silence worth seeing). - The chart stacks certain (`--accent`) and uncertain (`--text-muted`) — existing theme variables, so no new customizer entry. Descriptions updated. ## Performance `CertainActivity` walks a week of packets per watched node, at most once per node per 10 minutes, with the read lock taken **per node**: ``` BenchmarkCertainActivity-16 12.1 ms/op (50 watched repeaters × 6,000 packets; ~0.25 ms per lock hold) BenchmarkNoteCertainRelays-16 121 ns/op 0 allocs ``` Node analytics adds one `certainFor` per packet of the requested node (a prefix-map lookup, or the cached decoded JSON for its own packets). ## Tests - `adaptiveOfflineAfter`: busy repeater hits the floor; 5/hour; quiet nights set the rate; > 10 % empty hours, short history, too few observers → fixed. - `CertainActivity` ends at the last certain packet and counts only certain packets — verified red by mutating the window to end at now. - Notifier end to end: learned window passed → bad; inside → good; too few observers / setting off → fixed 72 h; cache refresh and prune. - `certainFor`, `longestSilence` (incl. running silence), config defaults, `/api/nodes/{pk}/analytics` splits a shared-prefix bucket. - Frontend: `NodeAnalyticsTimeline.series` (new `tests/unit/test-node-analytics-certain-timeline.js`). - `cmd/server` `go test ./...` and `test-all.sh` pass locally, also on a trial merge into a downstream fork. ## Validation Server on the freshened e2e fixture: analytics buckets carry `certain` (e.g. 89 of 90 packets certain for one repeater). **No browser screenshot**: Playwright isn't installed on this machine. ## Later (rule 8) `floorMinutes` / `minObservers` are server config, not theme settings; an admin UI for notification settings would be the place, not the customizer. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
387 lines
15 KiB
Go
387 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/meshcore-analyzer/users"
|
|
)
|
|
|
|
// Node notification evaluator (docs/specs/2026-10-07-node-notifications-design.md).
|
|
// evaluateNotifications is a pure function: everything it reads is in
|
|
// notifyInput, the time included, so it is tested without a clock or a
|
|
// database. The notifier (notify_service.go) gathers the input and acts on
|
|
// the result.
|
|
|
|
// notifyNode is one analyzer node row the evaluator needs.
|
|
type notifyNode struct {
|
|
Pubkey string // lowercase hex
|
|
Name string
|
|
Role string // as stored; compared lowercase
|
|
LastSeen string // nodes.last_seen, "" when NULL
|
|
BatteryMv *int // latest advert telemetry; nil when the node reports none
|
|
Foreign bool // nodes.foreign_advert = 1
|
|
}
|
|
|
|
// notifyObserver is one active (not soft-deleted) observer.
|
|
type notifyObserver struct {
|
|
ID string
|
|
Name string
|
|
LastSeen string // observers.last_seen, "" when NULL
|
|
}
|
|
|
|
// notifyInput is everything one evaluation reads.
|
|
type notifyInput struct {
|
|
Now time.Time
|
|
Health HealthThresholds // resolved, as Config.GetHealthThresholds returns it
|
|
LowMv int // Config.LowBatteryMv
|
|
Accounts map[int64]users.User
|
|
Prefs []users.NotifyPrefs
|
|
Watches []users.NotifyWatch
|
|
States []users.NotifyState
|
|
Nodes map[string]notifyNode // watched nodes, plus every foreign node when an admin chose foreign.new
|
|
Heard map[string]time.Time // newest packet certainly from the node (node page "Last Heard", #2260)
|
|
// SharedSeen is, per repeater and room, when its hop prefix (shared with
|
|
// other repeaters, last_relayed) was last in a path. Not evidence: it
|
|
// only explains an offline mail for a node that is "uncertain" in the UI.
|
|
SharedSeen map[string]time.Time
|
|
// OfflineAfter is the learned offline threshold per node (#2260,
|
|
// adaptiveOffline); a node without one, or with one beyond its role's
|
|
// silent window, uses that window.
|
|
OfflineAfter map[string]time.Duration
|
|
// NewestPacket is the newest packet in the store (zero: unknown). A
|
|
// node's silence is measured up to it, not up to Now: when the feed
|
|
// stalls, every node's clock stops with it, so a node with a short
|
|
// learned window is not mailed offline before IngestStale trips (#2260).
|
|
NewestPacket time.Time
|
|
Observers []notifyObserver // nil unless an admin chose observer.offline
|
|
// IngestStale: the newest packet in the store is old, so a silent node
|
|
// and a silent feed look the same. node.offline and observer.offline
|
|
// are then not compared (states unchanged, no changes).
|
|
IngestStale bool
|
|
// IngestResumedAt: when the last stale period ended (zero: none since
|
|
// startup). For a node or observer whose stored state is good, evidence
|
|
// older than this counts as heard at this time, so nothing goes offline
|
|
// until a full silent window has passed after the feed came back.
|
|
IngestResumedAt time.Time
|
|
// External is the fresh externalAlerts feed; nil when it is not
|
|
// configured, nobody chose node.external, or the feed is stale. nil
|
|
// leaves every node.external state unchanged.
|
|
External *externalFeed
|
|
}
|
|
|
|
// notifyChange is one transition for one user.
|
|
type notifyChange struct {
|
|
Event string
|
|
Subject string // node pubkey or observer id
|
|
Name string // display name at evaluation time
|
|
To string // users.NotifyGood, users.NotifyBad or users.NotifyTold
|
|
BatteryMv *int // node.battery only
|
|
// SharedSeen: node.offline only, when the node's shared hop prefix was
|
|
// still in a path after its last certain packet (zero otherwise).
|
|
SharedSeen time.Time
|
|
Text string // node.external only: the alert text ("" when resolved)
|
|
URL string // node.external only: the alert's link, "" for the node page
|
|
At time.Time
|
|
}
|
|
|
|
// notifyResult: Changes per user, ordered by event then name; States are
|
|
// the rows to write (new or changed), disabled users included; Drop are the
|
|
// rows to delete (admin-event rows of users who are no longer admin).
|
|
type notifyResult struct {
|
|
Changes map[int64][]notifyChange
|
|
States []users.NotifyState
|
|
Drop []users.NotifyKey
|
|
}
|
|
|
|
const (
|
|
// batteryHysteresisMv: a low battery is good again only at lowMv + 100.
|
|
batteryHysteresisMv = 100
|
|
// foreignBaselineSubject marks that an admin's foreign.new baseline
|
|
// exists: the first evaluation after opting in stores every current
|
|
// foreign node as told without mailing; afterwards a foreign node
|
|
// without a row is new.
|
|
foreignBaselineSubject = "*"
|
|
)
|
|
|
|
var notifyEventRank = map[string]int{
|
|
users.NotifyNodeOffline: 0, users.NotifyNodeBattery: 1, users.NotifyNodeExternal: 2,
|
|
users.NotifyForeignNew: 3, users.NotifyObserverOffline: 4,
|
|
}
|
|
|
|
// nodeLabel is the node's name made safe for a mail line (mailSafeText),
|
|
// or the first 8 hex characters of its key.
|
|
func nodeLabel(n notifyNode, known bool, pk string) string {
|
|
if name := mailSafeText(n.Name); known && name != "" {
|
|
return name
|
|
}
|
|
if len(pk) > 8 {
|
|
return pk[:8]
|
|
}
|
|
return pk
|
|
}
|
|
|
|
// nodeOnlineState is good when the node was heard within its role's silent
|
|
// window, like roles.js getNodeStatus: the latest of last_seen and the packet
|
|
// store's certain last heard (its own packets and relay hops that can only be
|
|
// this node, #2260). last_relayed is not evidence: it also counts 1-byte
|
|
// prefix matches shared with other repeaters, which kept a silent repeater
|
|
// "online" for days. The window is the role's silent window, or the node's
|
|
// learned OfflineAfter when that is shorter (adaptiveOffline). A node
|
|
// missing from the analyzer database is bad. While prev is good, the
|
|
// latest evidence is never older than in.IngestResumedAt (the outage grace).
|
|
// While prev is bad (entered at prevAt), the node comes back only on
|
|
// evidence from that moment on: a window that grows back (a learned
|
|
// threshold lost to retention or a restart) must not read a still-silent
|
|
// node as back online.
|
|
func nodeOnlineState(pk string, n notifyNode, known bool, prev string, prevAt time.Time, in *notifyInput) string {
|
|
if !known {
|
|
return users.NotifyBad
|
|
}
|
|
last, _ := parseRelayTS(n.LastSeen)
|
|
if t, ok := in.Heard[pk]; ok && t.After(last) {
|
|
last = t
|
|
}
|
|
if prev == users.NotifyGood && last.Before(in.IngestResumedAt) {
|
|
last = in.IngestResumedAt
|
|
}
|
|
if prev == users.NotifyBad && !prevAt.IsZero() && last.Before(prevAt) {
|
|
return users.NotifyBad
|
|
}
|
|
now := in.Now
|
|
if !in.NewestPacket.IsZero() && in.NewestPacket.Before(now) {
|
|
now = in.NewestPacket
|
|
}
|
|
_, silentMs := in.Health.GetHealthMs(strings.ToLower(n.Role))
|
|
silent := time.Duration(silentMs) * time.Millisecond
|
|
if a, ok := in.OfflineAfter[pk]; ok && a > 0 && a < silent {
|
|
silent = a
|
|
}
|
|
if last.IsZero() || now.Sub(last) >= silent {
|
|
return users.NotifyBad
|
|
}
|
|
return users.NotifyGood
|
|
}
|
|
|
|
// batteryState applies the low-battery threshold with hysteresis.
|
|
func batteryState(mv, lowMv int, prev string) string {
|
|
if prev == users.NotifyBad {
|
|
if mv >= lowMv+batteryHysteresisMv {
|
|
return users.NotifyGood
|
|
}
|
|
return users.NotifyBad
|
|
}
|
|
if mv < lowMv {
|
|
return users.NotifyBad
|
|
}
|
|
return users.NotifyGood
|
|
}
|
|
|
|
// observerState: bad when last_seen is older than observerStaleMinutes,
|
|
// good again within observerOnlineMinutes. While prev is good, last_seen
|
|
// is never older than resumed (the outage grace, zero for none).
|
|
func observerState(o notifyObserver, h HealthThresholds, now, resumed time.Time, prev string) string {
|
|
t, ok := parseRelayTS(o.LastSeen)
|
|
if prev == users.NotifyGood && !resumed.IsZero() && (!ok || t.Before(resumed)) {
|
|
t, ok = resumed, true
|
|
}
|
|
if !ok {
|
|
return users.NotifyBad
|
|
}
|
|
age := now.Sub(t)
|
|
if prev == users.NotifyBad {
|
|
if age <= time.Duration(h.ObserverOnlineMinutes)*time.Minute {
|
|
return users.NotifyGood
|
|
}
|
|
return users.NotifyBad
|
|
}
|
|
if age > time.Duration(h.ObserverStaleMinutes)*time.Minute {
|
|
return users.NotifyBad
|
|
}
|
|
return users.NotifyGood
|
|
}
|
|
|
|
// evalExternal compares one watched node's current external alerts with the
|
|
// stored node.external states of one user. The first fresh feed after the
|
|
// node is watched (no baseline row) stores the current alerts silently;
|
|
// afterwards a new or returning alert is a bad change and a stored bad alert
|
|
// missing from the feed is a good (resolved) change.
|
|
func evalExternal(res *notifyResult, prev map[users.NotifyKey]string, bad []string, current map[string]externalAlert,
|
|
uid int64, pk, name string, mailing bool, now time.Time) {
|
|
base := users.NotifyKey{UserID: uid, Event: users.NotifyNodeExternal, Subject: pk + "/" + externalBaselineKey}
|
|
_, baselined := prev[base]
|
|
if !baselined {
|
|
res.States = append(res.States, users.NotifyState{NotifyKey: base, State: users.NotifyGood, ChangedAt: now})
|
|
}
|
|
emit := func(key, state string, a externalAlert) {
|
|
k := users.NotifyKey{UserID: uid, Event: users.NotifyNodeExternal, Subject: pk + "/" + key}
|
|
res.States = append(res.States, users.NotifyState{NotifyKey: k, State: state, ChangedAt: now})
|
|
if baselined && mailing {
|
|
res.Changes[uid] = append(res.Changes[uid], notifyChange{Event: users.NotifyNodeExternal, Subject: k.Subject,
|
|
Name: name, To: state, Text: a.Text, URL: a.URL, At: now})
|
|
}
|
|
}
|
|
for _, key := range sortedExternalKeys(current) {
|
|
if prev[users.NotifyKey{UserID: uid, Event: users.NotifyNodeExternal, Subject: pk + "/" + key}] != users.NotifyBad {
|
|
emit(key, users.NotifyBad, current[key])
|
|
}
|
|
}
|
|
sort.Strings(bad)
|
|
for _, key := range bad {
|
|
if _, still := current[key]; !still {
|
|
emit(key, users.NotifyGood, externalAlert{})
|
|
}
|
|
}
|
|
}
|
|
|
|
// evaluateNotifications computes every chosen (user, event, subject) state
|
|
// and compares it with the stored one. A subject without a stored state is
|
|
// stored without a change (a restart or a new watch never mails); a stored
|
|
// state that differs is a change. Users missing from Accounts are skipped.
|
|
// A user with watches but no preferences row gets the defaults (enabled,
|
|
// node events). A disabled user's states are kept current without changes,
|
|
// so turning notifications on again never mails what happened while off.
|
|
// Admin events are evaluated only for admins; a non-admin's admin-event
|
|
// rows are dropped, like opting out, so a re-promotion starts silently.
|
|
func evaluateNotifications(in notifyInput) notifyResult {
|
|
prev := make(map[users.NotifyKey]string, len(in.States))
|
|
prevAt := make(map[users.NotifyKey]time.Time, len(in.States)) // when each stored state was entered
|
|
// extBad: per user and node, the node.external keys stored as bad.
|
|
extBad := map[int64]map[string][]string{}
|
|
for _, s := range in.States {
|
|
prev[s.NotifyKey] = s.State
|
|
prevAt[s.NotifyKey] = s.ChangedAt
|
|
if s.Event == users.NotifyNodeExternal && s.State == users.NotifyBad {
|
|
if pk, key, ok := strings.Cut(s.Subject, "/"); ok && key != externalBaselineKey {
|
|
if extBad[s.UserID] == nil {
|
|
extBad[s.UserID] = map[string][]string{}
|
|
}
|
|
extBad[s.UserID][pk] = append(extBad[s.UserID][pk], key)
|
|
}
|
|
}
|
|
}
|
|
watched := map[int64][]string{}
|
|
for _, w := range in.Watches {
|
|
watched[w.UserID] = append(watched[w.UserID], w.Pubkey)
|
|
}
|
|
var foreign []string
|
|
for pk, n := range in.Nodes {
|
|
if n.Foreign {
|
|
foreign = append(foreign, pk)
|
|
}
|
|
}
|
|
sort.Strings(foreign)
|
|
|
|
res := notifyResult{Changes: map[int64][]notifyChange{}}
|
|
for _, s := range in.States {
|
|
if u, ok := in.Accounts[s.UserID]; ok && u.Role != users.RoleAdmin && users.IsAdminNotifyEvent(s.Event) {
|
|
res.Drop = append(res.Drop, s.NotifyKey)
|
|
}
|
|
}
|
|
prefs := make([]users.NotifyPrefs, 0, len(in.Prefs)+len(watched))
|
|
hasPrefs := make(map[int64]bool, len(in.Prefs))
|
|
for _, p := range in.Prefs {
|
|
prefs = append(prefs, p)
|
|
hasPrefs[p.UserID] = true
|
|
}
|
|
var defaulted []int64
|
|
for uid := range watched {
|
|
if !hasPrefs[uid] {
|
|
defaulted = append(defaulted, uid)
|
|
}
|
|
}
|
|
sort.Slice(defaulted, func(i, j int) bool { return defaulted[i] < defaulted[j] })
|
|
for _, uid := range defaulted {
|
|
prefs = append(prefs, users.DefaultNotifyPrefs(uid))
|
|
}
|
|
mailing := false // Enabled of the user being evaluated
|
|
store := func(k users.NotifyKey, state string) {
|
|
res.States = append(res.States, users.NotifyState{NotifyKey: k, State: state, ChangedAt: in.Now})
|
|
}
|
|
compare := func(k users.NotifyKey, name, state string, mv *int) {
|
|
old, had := prev[k]
|
|
if had && old == state {
|
|
return
|
|
}
|
|
store(k, state)
|
|
if had && mailing {
|
|
res.Changes[k.UserID] = append(res.Changes[k.UserID],
|
|
notifyChange{Event: k.Event, Subject: k.Subject, Name: name, To: state, BatteryMv: mv, At: in.Now})
|
|
}
|
|
}
|
|
|
|
for _, p := range prefs {
|
|
u, ok := in.Accounts[p.UserID]
|
|
if !ok {
|
|
continue
|
|
}
|
|
mailing = p.Enabled
|
|
for _, pk := range watched[p.UserID] {
|
|
n, known := in.Nodes[pk]
|
|
name := nodeLabel(n, known, pk)
|
|
if p.Has(users.NotifyNodeOffline) && !in.IngestStale {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyNodeOffline, Subject: pk}
|
|
state := nodeOnlineState(pk, n, known, prev[k], prevAt[k], &in)
|
|
before := len(res.Changes[p.UserID])
|
|
compare(k, name, state, nil)
|
|
if state == users.NotifyBad && len(res.Changes[p.UserID]) > before {
|
|
if seen, ok := in.SharedSeen[pk]; ok && seen.After(in.Heard[pk]) {
|
|
res.Changes[p.UserID][before].SharedSeen = seen
|
|
}
|
|
}
|
|
}
|
|
if p.Has(users.NotifyNodeBattery) && known && n.BatteryMv != nil {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyNodeBattery, Subject: pk}
|
|
compare(k, name, batteryState(*n.BatteryMv, in.LowMv, prev[k]), n.BatteryMv)
|
|
}
|
|
if p.Has(users.NotifyNodeExternal) && in.External != nil {
|
|
evalExternal(&res, prev, extBad[p.UserID][pk], in.External.Alerts[pk], p.UserID, pk, name, mailing, in.Now)
|
|
}
|
|
}
|
|
if u.Role != users.RoleAdmin {
|
|
continue
|
|
}
|
|
if p.Has(users.NotifyForeignNew) {
|
|
base := users.NotifyKey{UserID: p.UserID, Event: users.NotifyForeignNew, Subject: foreignBaselineSubject}
|
|
_, baselined := prev[base]
|
|
if !baselined {
|
|
store(base, users.NotifyGood)
|
|
}
|
|
for _, pk := range foreign {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyForeignNew, Subject: pk}
|
|
if _, told := prev[k]; told {
|
|
continue
|
|
}
|
|
store(k, users.NotifyTold)
|
|
if baselined && mailing {
|
|
res.Changes[p.UserID] = append(res.Changes[p.UserID], notifyChange{Event: users.NotifyForeignNew, Subject: pk,
|
|
Name: nodeLabel(in.Nodes[pk], true, pk), To: users.NotifyTold, At: in.Now})
|
|
}
|
|
}
|
|
}
|
|
if p.Has(users.NotifyObserverOffline) && !in.IngestStale {
|
|
for _, o := range in.Observers {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyObserverOffline, Subject: o.ID}
|
|
name := mailSafeText(o.Name)
|
|
if name == "" {
|
|
name = o.ID
|
|
}
|
|
compare(k, name, observerState(o, in.Health, in.Now, in.IngestResumedAt, prev[k]), nil)
|
|
}
|
|
}
|
|
}
|
|
for _, list := range res.Changes {
|
|
sort.SliceStable(list, func(i, j int) bool {
|
|
if a, b := notifyEventRank[list[i].Event], notifyEventRank[list[j].Event]; a != b {
|
|
return a < b
|
|
}
|
|
if list[i].Name != list[j].Name {
|
|
return list[i].Name < list[j].Name
|
|
}
|
|
return list[i].Subject < list[j].Subject
|
|
})
|
|
}
|
|
return res
|
|
}
|