From fbff628926d5c1d8b5bf641607f911a6c10bd6f5 Mon Sep 17 00:00:00 2001 From: Jonathon Leight Date: Sat, 8 Aug 2026 20:54:03 -0400 Subject: [PATCH] Split out probe traffic in analytics page --- internal/analytics/analytics.go | 23 +-- internal/analytics/analytics_test.go | 66 +++++- internal/analytics/classify.go | 190 ++++++++++++++++++ internal/analytics/classify_test.go | 187 +++++++++++++++++ internal/core/analytics.go | 75 +++++-- internal/core/templates/analytics.html | 70 +++++++ internal/e2e/analytics_cards_test.go | 91 +++++++++ internal/marketing/templates/privacy.html | 6 +- internal/store/analytics.go | 97 ++++++--- internal/store/analytics_test.go | 170 ++++++++++++++-- .../store/migrations/0045_analytics_kind.sql | 95 +++++++++ 11 files changed, 987 insertions(+), 83 deletions(-) create mode 100644 internal/analytics/classify.go create mode 100644 internal/analytics/classify_test.go create mode 100644 internal/e2e/analytics_cards_test.go create mode 100644 internal/store/migrations/0045_analytics_kind.sql diff --git a/internal/analytics/analytics.go b/internal/analytics/analytics.go index 0597a46..9cb057d 100644 --- a/internal/analytics/analytics.go +++ b/internal/analytics/analytics.go @@ -132,13 +132,15 @@ func (rec *Recorder) Handler(next http.Handler) http.Handler { // buffer is full the event is dropped rather than slowing the response. func (rec *Recorder) record(r *http.Request, status int) { host := web.HostWithoutPort(r.Host) + path := web.RedactPath(r.URL.Path) ev := store.AnalyticsEvent{ Ts: time.Now(), Surface: rec.surface(host), Host: host, - Path: web.RedactPath(r.URL.Path), + Path: path, Method: r.Method, Status: status, + Kind: classify(path, status, r.UserAgent()), Visitor: rec.visitor(r), } select { @@ -174,8 +176,10 @@ func (rec *Recorder) visitor(r *http.Request) string { return hex.EncodeToString(h.Sum(nil)[:8]) } -// skip drops health checks, static assets, websockets, preflight/HEAD, and bots — -// "people visiting", not infrastructure noise. +// skip drops health checks, static assets, websockets, and preflight/HEAD — +// infrastructure that isn't a request for a page. Bots and scanners are no +// longer dropped here: they're recorded under their own kind (see classify) so +// their volume is visible without being counted as visits. func skip(r *http.Request) bool { if r.Method == http.MethodOptions || r.Method == http.MethodHead { return true @@ -192,19 +196,6 @@ func skip(r *http.Request) bool { strings.HasSuffix(p, "/ws"): return true } - return isBot(r.UserAgent()) -} - -func isBot(ua string) bool { - ua = strings.ToLower(ua) - if ua == "" { - return true - } - for _, s := range []string{"bot", "crawl", "spider", "slurp", "headless", "preview", "monitor"} { - if strings.Contains(ua, s) { - return true - } - } return false } diff --git a/internal/analytics/analytics_test.go b/internal/analytics/analytics_test.go index abf52e1..d454ef5 100644 --- a/internal/analytics/analytics_test.go +++ b/internal/analytics/analytics_test.go @@ -61,8 +61,6 @@ func TestHandlerSkips(t *testing.T) { {"/healthz", "Mozilla/5.0"}, {"/static/app.css", "Mozilla/5.0"}, {"/repeaters/abc/ws", "Mozilla/5.0"}, - {"/dashboard", "Googlebot/2.1"}, // bot UA - {"/dashboard", ""}, // empty UA } for _, c := range cases { req := httptest.NewRequest(http.MethodGet, "http://app.x"+c.path, nil) @@ -78,6 +76,70 @@ func TestHandlerSkips(t *testing.T) { } } +// TestHandlerRecordsKind covers the whole point of the kind column: scanners and +// crawlers reach the store rather than being dropped at the door, but under a +// kind the dashboard can hold apart from real visits. +func TestHandlerRecordsKind(t *testing.T) { + cases := []struct { + name, path, ua string + status int + want string + }{ + {"visit", "/dashboard", "Mozilla/5.0", http.StatusOK, KindVisit}, + {"probe", "/wp-login.php", "Mozilla/5.0", http.StatusNotFound, KindProbe}, + {"bot", "/orgs", "Googlebot/2.1", http.StatusOK, KindBot}, + {"empty ua is a bot", "/orgs", "", http.StatusOK, KindBot}, + {"broken link", "/orgs/gone", "Mozilla/5.0", http.StatusNotFound, KindNotFound}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + rec := testRecorder() + h := rec.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(c.status) + })) + req := httptest.NewRequest(http.MethodGet, "http://app.x"+c.path, nil) + if c.ua != "" { + req.Header.Set("User-Agent", c.ua) + } + h.ServeHTTP(httptest.NewRecorder(), req) + + select { + case e := <-rec.ch: + if e.Kind != c.want { + t.Fatalf("kind = %q, want %q (event %+v)", e.Kind, c.want, e) + } + default: + t.Fatalf("%s was dropped; it should be recorded as %q", c.name, c.want) + } + }) + } +} + +// TestKindUsesRedactedPath: classification runs on the path as stored, so a +// secret in the URL can't reach the classifier's signature lists either. +func TestKindUsesRedactedPath(t *testing.T) { + rec := testRecorder() + h := rec.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", http.StatusNotFound) + })) + + req := httptest.NewRequest(http.MethodGet, "http://app.x/invite/S3cr3tShareToken.php", nil) + req.Header.Set("User-Agent", "Mozilla/5.0") + h.ServeHTTP(httptest.NewRecorder(), req) + + select { + case e := <-rec.ch: + if e.Path != "/invite/:token" { + t.Fatalf("recorded path = %q, want the token redacted", e.Path) + } + if e.Kind != KindNotFound { + t.Fatalf("kind = %q, want %q — the .php was in the redacted-away token", e.Kind, KindNotFound) + } + default: + t.Fatal("expected an event to be enqueued") + } +} + func TestSurfaceClassification(t *testing.T) { rec := testRecorder() for host, want := range map[string]string{ diff --git a/internal/analytics/classify.go b/internal/analytics/classify.go new file mode 100644 index 0000000..94ad8d9 --- /dev/null +++ b/internal/analytics/classify.go @@ -0,0 +1,190 @@ +package analytics + +import "strings" + +// Event kinds. Every recorded request gets exactly one, decided at record time +// by classify. They're stored on the row and on every rollup so the dashboard +// can read one kind at a time — scanner noise is kept, not discarded, but it +// never lands in the same bucket as a person visiting a page. +const ( + KindVisit = "visit" // a real request that hit a real route + KindProbe = "probe" // a request for something only an attacker asks for + KindNotFound = "notfound" // a 404 with no attack signature — likely a broken link + KindBot = "bot" // a self-identified crawler or monitor +) + +// probeExtensions are file types we serve nowhere. Matched against every path +// SEGMENT, not just the last one: the Laravel Ignition RCE arrives as +// /index.php/_ignition/execute-solution, where the ".php" sits mid-path. +var probeExtensions = []string{ + ".php", ".phps", ".php3", ".php5", ".php7", + ".ini", ".env", ".yaml", ".yml", ".sql", ".py", ".tfstate", ".properties", + ".asp", ".aspx", ".axd", ".cgi", ".jsp", ".jspx", ".action", + // .js is safe to claim: the only JavaScript we serve lives under /static/, + // which skip() drops before anything is recorded, so a .js that reaches the + // classifier is by definition not ours. + ".js", + // Editor and backup droppings. These also arrive appended to a real + // extension (/phpinfo.php.save, /config.json.save), which is why the check + // below strips them and re-tests rather than only matching the tail. + ".bak", ".old", ".swp", ".save", ".orig", ".copy", ".dist", ".tmp", +} + +// backupSuffixes get stripped from a segment before the extension check runs, so +// /phpinfo.php~ is recognized as the .php probe it is. +var backupSuffixes = []string{"~", ".save", ".bak", ".old", ".orig", ".copy", ".dist", ".tmp", ".backup"} + +// probeNames match a path segment exactly — credential and config files that +// ship with other stacks. Matching the whole segment (never a substring) is what +// keeps our own /orgs/{id}/repeaters.json and /repeaters/{id}/config.json out of +// this bucket; note that bare "config.json" is deliberately absent for exactly +// that reason and is handled by rootProbes instead. +var probeNames = []string{ + "firebase-key.json", "credentials.json", "service-account.json", + "secrets.json", "settings.json", "appsettings.json", "sftp.json", + "package.json", "composer.json", "web.config", + "id_rsa", "id_dsa", "backup.zip", "backup.tar.gz", "dockerfile", +} + +// jsonRoots are the only path prefixes under which we serve JSON +// (/orgs/{id}/repeaters.json and /repeaters/{id}/config.json). A .json anywhere +// else is someone fishing for another stack's credentials — production alone +// turned up gcp-credentials.json, firebase-adminsdk.json, aws-ses.json and +// appsettings.Production.json, which no fixed list of names would have kept up +// with. Scoping by prefix rather than by name keeps a 404 on our own two +// endpoints readable as the broken link it is. +var jsonRoots = []string{"/orgs/", "/repeaters/"} + +// rootProbes are generic names that are only suspicious at the root of a host. +// "/api" and "/console" are scanner bait; /api/login/begin and +// /repeaters/{id}/console are ours. Exact full-path matches only. +var rootProbes = []string{ + "/api", "/info", "/env", "/server", "/phpinfo", "/console", "/console/", + "/config.json", "/config.js", "/aws.config.js", + "/server-status", "/server-info", "/v2/_catalog", "/old/", +} + +// probeSegments are fragments from the standard scanner wordlists — other +// stacks' admin panels, framework internals, and known RCE entry points. Each is +// specific enough not to collide with our own URL space. +var probeSegments = []string{ + "wp-", "wordpress", "xmlrpc", "phpmyadmin", "/pma/", "adminer", "cgi-bin", + "/vendor/", "autodiscover", "/owa/", "/ecp/", "manager/html", "/solr/", + "jenkins", "actuator", "telescope", "eval-stdin", "hnap1", + "graphql", "/gql", "_profiler", "@vite", "___proxy_subdomain", + "debug/default", "_catalog", "_ignition", "webhook-waiting", + "stats/prometheus", "/goform/", "/boaform/", "_environment", "meta-inf", +} + +// classify buckets one finished request. +// +// A probe is a request for something we don't serve, that only an attacker asks +// for. The status gate is "not a 2xx": a scanner sweeping the www host gets a +// 301 to the apex rather than a 404, and gating on 404 alone let all of that +// through as ordinary traffic. A 2xx means we really do serve the path, so the +// signature must be wrong and the request stays a visit — that direction is the +// safe one to be wrong in. +// +// The signatures only decide which flavor of non-2xx it was, so a miss can never +// hide a real request: it degrades to "notfound", still visible, just not +// attributed to an attacker. Bots are checked after probes because a scanner is +// free to put "bot" in its user agent, and what it asked for is better evidence +// than what it calls itself. +// +// The SQL backfill in migration 0045 mirrors these rules; it runs once over +// history and the two aren't kept in lockstep afterwards. +func classify(path string, status int, ua string) string { + if !isSuccess(status) && isProbePath(path) { + return KindProbe + } + if isBot(ua) { + return KindBot + } + if status == 404 { + return KindNotFound + } + return KindVisit +} + +// isSuccess reports whether the response actually served the path. +func isSuccess(status int) bool { return status >= 200 && status < 300 } + +// isProbePath reports whether a path looks like it came off a scanner wordlist. +// Case-insensitive: the same list gets replayed in every casing. +func isProbePath(path string) bool { + p := strings.ToLower(path) + + // A literal "*" is an unfilled placeholder from the scanner's own template + // (/workspaces/*, /webhook-waiting/*). No browser ever sends one. + if strings.Contains(p, "*") { + return true + } + for _, s := range rootProbes { + if p == s { + return true + } + } + for _, s := range probeSegments { + if strings.Contains(p, s) { + return true + } + } + jsonIsOurs := false + for _, root := range jsonRoots { + if strings.HasPrefix(p, root) { + jsonIsOurs = true + break + } + } + + for _, seg := range strings.Split(p, "/") { + if seg == "" { + continue + } + // We serve no dotfiles. .well-known is the one real convention, and + // exempting it keeps security.txt and friends out of the attack bucket. + if seg[0] == '.' && seg != ".well-known" { + return true + } + if strings.HasSuffix(seg, ".json") && !jsonIsOurs { + return true + } + for _, s := range probeNames { + if seg == s { + return true + } + } + // Strip editor/backup droppings before testing the extension, so + // /phpinfo.php.save and /phpinfo.php~ read as the .php probes they are. + base := seg + for changed := true; changed; { + changed = false + for _, s := range backupSuffixes { + if trimmed, ok := strings.CutSuffix(base, s); ok && trimmed != "" { + base, changed = trimmed, true + } + } + } + for _, s := range probeExtensions { + if strings.HasSuffix(seg, s) || strings.HasSuffix(base, s) { + return true + } + } + } + return false +} + +// isBot reports whether the user agent identifies itself as automated. An empty +// user agent counts: every real browser sends one. +func isBot(ua string) bool { + ua = strings.ToLower(ua) + if ua == "" { + return true + } + for _, s := range []string{"bot", "crawl", "spider", "slurp", "headless", "preview", "monitor"} { + if strings.Contains(ua, s) { + return true + } + } + return false +} diff --git a/internal/analytics/classify_test.go b/internal/analytics/classify_test.go new file mode 100644 index 0000000..2d6e5ac --- /dev/null +++ b/internal/analytics/classify_test.go @@ -0,0 +1,187 @@ +package analytics + +import "testing" + +const browserUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" + +func TestClassify(t *testing.T) { + cases := []struct { + name string + path string + status int + ua string + want string + }{ + // Real traffic. + {"page view", "/orgs/example-mesh", 200, browserUA, KindVisit}, + {"redirect", "/", 301, browserUA, KindVisit}, + {"form post", "/orgs/new", 302, browserUA, KindVisit}, + {"server error is still a visit", "/dashboard", 500, browserUA, KindVisit}, + {"forbidden is still a visit", "/admin/users", 403, browserUA, KindVisit}, + + // Scanner traffic, all pulled from production logs. Every one arrived with + // a browser user agent, which is why the UA check alone never caught them. + {"php config", "/wp_mail_smtp.ini", 404, browserUA, KindProbe}, + {"php index", "/index.php", 404, browserUA, KindProbe}, + {"phpinfo", "/includes/phpinfo.php", 404, browserUA, KindProbe}, + {"firebase key", "/firebase-key.json", 404, browserUA, KindProbe}, + {"cli config", "/.vultr-cli.yaml", 404, browserUA, KindProbe}, + {"tomcat manager", "/manager/html", 404, browserUA, KindProbe}, + {"dotenv", "/.env", 404, browserUA, KindProbe}, + {"git config", "/.git/config", 404, browserUA, KindProbe}, + {"wordpress login", "/wp-login.php", 404, browserUA, KindProbe}, + {"uppercase replay", "/WP-ADMIN/SETUP-CONFIG.PHP", 404, browserUA, KindProbe}, + + // Missed by the first version of this list — the reason it was rewritten. + {"laravel ignition rce", "/index.php/_ignition/execute-solution", 404, browserUA, KindProbe}, + {"graphql", "/graphql", 404, browserUA, KindProbe}, + {"graphql under api", "/api/graphql", 404, browserUA, KindProbe}, + {"apache status", "/server-status", 404, browserUA, KindProbe}, + {"docker registry", "/v2/_catalog", 404, browserUA, KindProbe}, + {"vite env", "/@vite/env", 404, browserUA, KindProbe}, + {"aspnet trace", "/trace.axd", 404, browserUA, KindProbe}, + {"struts action", "/login.action", 404, browserUA, KindProbe}, + {"cpanel proxy", "/___proxy_subdomain_cpanel", 404, browserUA, KindProbe}, + {"symfony profiler", "/_profiler/phpinfo", 404, browserUA, KindProbe}, + {"ds_store", "/.DS_Store", 404, browserUA, KindProbe}, + {"stripe dotfile", "/.stripe/", 404, browserUA, KindProbe}, + {"bare api", "/api", 404, browserUA, KindProbe}, + {"bare console", "/console/", 404, browserUA, KindProbe}, + {"root config.json", "/config.json", 404, browserUA, KindProbe}, + {"unfilled wildcard", "/workspaces/*", 404, browserUA, KindProbe}, + {"unfilled wildcard 2", "/webhook-waiting/*", 404, browserUA, KindProbe}, + + // .env variants — an exact-name list can't keep up, the dotfile rule can. + {"env tilde", "/.env~", 404, browserUA, KindProbe}, + {"env copy", "/.env_copy", 404, browserUA, KindProbe}, + {"env backup2", "/.env.backup2", 404, browserUA, KindProbe}, + {"nested env", "/backend/.env", 404, browserUA, KindProbe}, + + // A scanner sweeping the www host gets a 301, not a 404. Gating probes on + // 404 alone filed all of this as ordinary traffic. + {"probe redirected by www host", "/.env", 301, browserUA, KindProbe}, + {"probe on a 500", "/wp-login.php", 500, browserUA, KindProbe}, + + // Bots. + {"googlebot", "/orgs", 200, "Googlebot/2.1 (+http://www.google.com/bot.html)", KindBot}, + {"empty ua", "/orgs", 200, "", KindBot}, + {"uptime monitor", "/", 200, "Better Uptime Monitor", KindBot}, + + // A scanner that also calls itself a bot is still a scanner. + {"bot ua on a probe path", "/wp-login.php", 404, "evilbot/1.0", KindProbe}, + + // Honest 404s stay visible as their own kind rather than being written off + // as attacks — a broken link is something worth fixing. + {"stale link", "/orgs/closed-club", 404, browserUA, KindNotFound}, + {"typo", "/repeaterz", 404, browserUA, KindNotFound}, + {"crawler on a dead link", "/orgs/gone", 404, "Googlebot/2.1", KindBot}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := classify(c.path, c.status, c.ua); got != c.want { + t.Errorf("classify(%q, %d, %q) = %q, want %q", c.path, c.status, c.ua, got, c.want) + } + }) + } +} + +// TestClassifyProductionCorpus runs the classifier over paths taken verbatim +// from production traffic. The same corpus was checked against migration 0045's +// SQL backfill, which found zero disagreements — keeping this list here is what +// stops the Go rules and the SQL rules drifting apart unnoticed. +func TestClassifyProductionCorpus(t *testing.T) { + probes := []string{ + "/xmlrpc.php", "/info.php", "/test.php", "/.aws/credentials", "/actuator/env", + "/.vscode/sftp.json", "/api/gql", "/graphql/api", "/debug/default/view", + "/___proxy_subdomain_whm/login", "/stats/prometheus", "/.linode-cli", + "/config.js", "/aws.config.js", "/info", "/env", "/server", "/phpinfo", + "/.env.save", "/.env.bak", "/.env.prod", "/.env.dev", "/.env.old", "/.env.example", + "/app/.env", "/api/.env", "/backend/.env", + "/ecp/Current/exporttool/microsoft.exchange.ediscovery.exporttool.application", + // Backup/editor suffixes appended AFTER a real extension. + "/phpinfo.php~", "/phpinfo.php.save", "/config.json.save", + // Credential JSON under other stacks' names — no fixed list keeps up, + // which is why .json is scoped by prefix instead. + "/gcp-credentials.json", "/google-credentials.json", "/firebase-adminsdk.json", + "/aws-ses.json", "/aws.json", "/env.json", "/config/production.json", + "/appsettings.Development.json", "/appsettings.Production.json", + // Loose source and infra files. + "/aws-config.js", "/app.js", "/index.js", "/server.js", "/env.js", "/js/config.js", + "/settings.py", "/terraform.tfstate", "/Dockerfile", "/_environment", "/old/", + "/s/230313e28343e2333313e28363/_/;/META-INF/maven/com.atlassian.jira/jira-webapp-dist/pom.properties", + } + for _, p := range probes { + if got := classify(p, 404, browserUA); got != KindProbe { + t.Errorf("classify(%q, 404) = %q, want %q", p, got, KindProbe) + } + } + + // Seen in production 404s and NOT attacks: these must stay visible as broken + // links so they can be fixed. + // Real people looking for pages we don't have, and crawlers on the auth host. + // These must stay readable as gaps to fill, not get written off as attacks. + notFound := []string{ + "/about", "/contact", "/contact-us", "/login", + "/login/sitemap.xml", "/login/robots.txt", + "/orgs/example-mesh/config/edit", "/repeaters/Nf5YgD1sJw6k/console", + } + for _, p := range notFound { + if got := classify(p, 404, browserUA); got != KindNotFound { + t.Errorf("classify(%q, 404) = %q, want %q", p, got, KindNotFound) + } + } +} + +// TestOwnRoutesAreNotProbes guards the signature list against our own URL space. +// The rules match extensions, filenames, and dotfiles, and MeshTender genuinely +// serves .json endpoints, has "config" all through the org routes, and has a +// /console under each repeater — a loose rule here would file a member's stale +// bookmark under "attacker". +func TestOwnRoutesAreNotProbes(t *testing.T) { + ours := []string{ + "/orgs/example-mesh/repeaters.json", + "/repeaters/Ab3xKp9QmR2t/config.json", + "/orgs/example-mesh/config", + "/orgs/example-mesh/config/profiles/new", + "/orgs/example-mesh/config/regions/12/area", + "/orgs/example-mesh/config/root-flood", + "/orgs/example-mesh/my-commands", + "/repeaters/Ab3xKp9QmR2t/console", + "/repeaters/Zq7WnT4vLh8c/console", + "/account/passkeys/rename", + "/api/login/discoverable/begin", + "/catalog/heltec-v3", + "/invite/:token", + "/build", + } + for _, p := range ours { + if isProbePath(p) { + t.Errorf("isProbePath(%q) = true, but that's one of our own routes", p) + } + if got := classify(p, 404, browserUA); got != KindNotFound { + t.Errorf("classify(%q, 404) = %q, want %q — a 404 on our own route is a broken link, not an attack", p, got, KindNotFound) + } + } +} + +// TestConventionalPathsAreNotProbes: these 404 in production today and are worth +// serving, not worth calling attacks. Filing them under "probe" would bury the +// signal that we ought to add them — real iOS devices and search engines are +// asking. .well-known in particular has to survive the dotfile rule. +func TestConventionalPathsAreNotProbes(t *testing.T) { + conventional := []string{ + "/apple-touch-icon.png", + "/apple-touch-icon-precomposed.png", + "/favicon.png", + "/sitemap.xml", + "/robots.txt", + "/llms.txt", + "/.well-known/security.txt", + "/.well-known/change-password", + } + for _, p := range conventional { + if isProbePath(p) { + t.Errorf("isProbePath(%q) = true, but that's a web convention worth serving", p) + } + } +} diff --git a/internal/core/analytics.go b/internal/core/analytics.go index 9597208..73e7450 100644 --- a/internal/core/analytics.go +++ b/internal/core/analytics.go @@ -3,6 +3,9 @@ package core import ( "net/http" "time" + + "github.com/jleight/meshtender/internal/analytics" + "github.com/jleight/meshtender/internal/store" ) // analyticsBar is one day's column in the traffic charts; heights are percentages @@ -33,31 +36,53 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) { days = 90 } - daily, err := s.Store.AnalyticsDaily(r.Context(), days) + // Everything on the main dashboard reads the "visit" kind only. Scanners and + // crawlers are recorded too (see internal/analytics classify), but they'd + // dwarf the real numbers here — they get their own cards below. + daily, err := s.Store.AnalyticsDaily(r.Context(), days, analytics.KindVisit) if err != nil { s.ServerError(w, r, "could not load analytics", err) return } - surfaces, err := s.Store.AnalyticsBySurface(r.Context(), days) + surfaces, err := s.Store.AnalyticsBySurface(r.Context(), days, analytics.KindVisit) if err != nil { s.ServerError(w, r, "could not load analytics", err) return } - paths, err := s.Store.AnalyticsTopPaths(r.Context(), days, 20) + paths, err := s.Store.AnalyticsTopPaths(r.Context(), days, analytics.KindVisit, 20) if err != nil { s.ServerError(w, r, "could not load analytics", err) return } - hosts, err := s.Store.AnalyticsTopHosts(r.Context(), days, 15) + hosts, err := s.Store.AnalyticsTopHosts(r.Context(), days, analytics.KindVisit, 15) if err != nil { s.ServerError(w, r, "could not load analytics", err) return } - visitors, err := s.Store.AnalyticsTopVisitors(r.Context(), days, 15) + visitors, err := s.Store.AnalyticsTopVisitors(r.Context(), days, analytics.KindVisit, 15) if err != nil { s.ServerError(w, r, "could not load analytics", err) return } + kinds, err := s.Store.AnalyticsKindSummary(r.Context(), days) + if err != nil { + s.ServerError(w, r, "could not load analytics", err) + return + } + probePaths, err := s.Store.AnalyticsTopPaths(r.Context(), days, analytics.KindProbe, 10) + if err != nil { + s.ServerError(w, r, "could not load analytics", err) + return + } + botPaths, err := s.Store.AnalyticsTopPaths(r.Context(), days, analytics.KindBot, 10) + if err != nil { + s.ServerError(w, r, "could not load analytics", err) + return + } + byKind := make(map[string]store.KindStat, len(kinds)) + for _, k := range kinds { + byKind[k.Kind] = k + } var maxReq, maxVis, totalReq int64 for _, d := range daily { @@ -101,16 +126,7 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) { surfaceRows = append(surfaceRows, analyticsRow{Label: x.Surface, Value: x.Requests, W: barPct(x.Requests, maxSurface)}) } - var maxHits int64 - for _, p := range paths { - if p.Hits > maxHits { - maxHits = p.Hits - } - } - pathRows := make([]analyticsRow, 0, len(paths)) - for _, p := range paths { - pathRows = append(pathRows, analyticsRow{Label: p.Path, Value: p.Hits, W: barPct(p.Hits, maxHits)}) - } + pathRows := toPathRows(paths) var maxHostReq int64 for _, h := range hosts { @@ -145,6 +161,7 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) { }) } + probes, bots := byKind[analytics.KindProbe], byKind[analytics.KindBot] s.Render(w, r, "analytics.html", map[string]any{ "Days": days, "Bars": bars, @@ -155,10 +172,36 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) { "TotalReq": totalReq, "TodayReq": todayReq, "TodayVis": todayVis, - "HasData": len(daily) > 0, + // Scanner and crawler traffic, held apart from the figures above so a + // wordlist replay can't read as an audience. + "ProbeReq": probes.Requests, + "ProbeSources": probes.Sources, + "ProbePaths": toPathRows(probePaths), + "BotReq": bots.Requests, + "BotSources": bots.Sources, + "BotPaths": toPathRows(botPaths), + // 404s with no attack signature — broken links worth fixing. Filtered out + // of every figure above, so without this they'd be invisible entirely. + "NotFoundReq": byKind[analytics.KindNotFound].Requests, + "HasData": len(daily) > 0 || len(kinds) > 0, }) } +// toPathRows scales a set of path counts into labeled bars. +func toPathRows(paths []store.PathStat) []analyticsRow { + var max int64 + for _, p := range paths { + if p.Hits > max { + max = p.Hits + } + } + rows := make([]analyticsRow, 0, len(paths)) + for _, p := range paths { + rows = append(rows, analyticsRow{Label: p.Path, Value: p.Hits, W: barPct(p.Hits, max)}) + } + return rows +} + // visitorRow is one (daily-rotating) visitor hash for the "traffic by user" table. type visitorRow struct { Visitor string diff --git a/internal/core/templates/analytics.html b/internal/core/templates/analytics.html index ccad38e..5419aab 100644 --- a/internal/core/templates/analytics.html +++ b/internal/core/templates/analytics.html @@ -43,6 +43,13 @@ + +
@@ -122,6 +129,69 @@
+ + +
+
+
+
+
+

Probes & scanners

+
Requests for things we don't serve — other stacks' admin panels, + credential files, known exploit paths.
+
+
+
+
+
+
{{.ProbeReq}}
+
Requests · last {{.Days}}d
+
+
+
{{.ProbeSources}}
+
Distinct sources
+
+
+ {{if .ProbePaths}}{{range .ProbePaths}}{{template "value-bar" .}}{{end}} + {{else}}
Nothing probed in this window.
{{end}} +
+
+
+
+
+
+
+

Bots & crawlers

+
Clients that identify themselves as automated: search engines, + uptime monitors, link previewers.
+
+
+
+
+
+
{{.BotReq}}
+
Requests · last {{.Days}}d
+
+
+
{{.BotSources}}
+
Distinct sources
+
+
+ {{if .BotPaths}}{{range .BotPaths}}{{template "value-bar" .}}{{end}} + {{else}}
No crawler traffic in this window. Bots are only recorded + from the point this was deployed.
{{end}} +
+
+
+
+ +
+
+
{{.NotFoundReq}}
+
404s with no attack signature · last {{.Days}}d — broken links worth fixing, + counted nowhere above.
+
+
{{end}} {{template "icon-arrow-left" "me-1"}}Back to admin {{end}} diff --git a/internal/e2e/analytics_cards_test.go b/internal/e2e/analytics_cards_test.go new file mode 100644 index 0000000..0c4538c --- /dev/null +++ b/internal/e2e/analytics_cards_test.go @@ -0,0 +1,91 @@ +//go:build browser + +package e2e + +import ( + "strings" + "testing" + "time" + + "github.com/chromedp/chromedp" + + "github.com/jleight/meshtender/internal/store" +) + +// TestAnalyticsKindCards drives the traffic dashboard in a real browser after +// seeding one of every event kind. It asserts the split the whole `kind` column +// exists for: a scanner replaying a wordlist is counted in its own card and is +// absent from the visitor and busiest-pages figures, where it used to outweigh +// every real reader. Also catches a CSP trip on the new markup, which no Go +// handler test can see. +func TestAnalyticsKindCards(t *testing.T) { + srv := newE2EServer(t) + // login()'s first account bootstraps the capabilities that reach /admin. + _, cookie := srv.login(t, "e2eadmin") + + now := time.Now() + ev := func(path, kind, visitor string, status int) store.AnalyticsEvent { + return store.AnalyticsEvent{ + Ts: now, Surface: "root", Host: "root.example", Path: path, + Method: "GET", Status: status, Kind: kind, Visitor: visitor, + } + } + evs := []store.AnalyticsEvent{ + ev("/", "visit", "alice", 200), + ev("/docs", "visit", "bob", 200), + ev("/gone", "notfound", "alice", 404), + ev("/orgs", "bot", "googlebot", 200), + ev("/catalog", "bot", "googlebot", 200), + } + // One scanner, five paths — the shape that swamped the dashboard. + for _, p := range []string{"/.env", "/wp-login.php", "/.git/config", "/index.php", "/phpinfo.php"} { + evs = append(evs, ev(p, "probe", "scanner", 404)) + } + if err := srv.store.InsertAnalyticsEvents(srv.ctx, evs); err != nil { + t.Fatalf("seed analytics events: %v", err) + } + // The path cards read the rollups, not raw events. + if err := srv.store.RollupAnalytics(srv.ctx); err != nil { + t.Fatalf("roll up analytics: %v", err) + } + + ctx, cancel, watch := startBrowser(t) + defer cancel() + + var probeReq, botReq, notFoundReq, totalReq, probeCard, pagesCard string + if err := chromedp.Run(ctx, + setSessionCookie(cookie), + chromedp.Navigate(srv.appURL+"/admin/analytics"), + chromedp.WaitVisible(`[data-testid="probe-card"]`, chromedp.ByQuery), + chromedp.Text(`[data-testid="probe-requests"]`, &probeReq, chromedp.ByQuery), + chromedp.Text(`[data-testid="bot-requests"]`, &botReq, chromedp.ByQuery), + chromedp.Text(`[data-testid="notfound-requests"]`, ¬FoundReq, chromedp.ByQuery), + chromedp.Text(`[data-testid="probe-card"]`, &probeCard, chromedp.ByQuery), + // The summary tile and the busiest-pages card must both be visits-only. + chromedp.Text(`.card-sm .h1`, &totalReq, chromedp.ByQuery), + chromedp.Text(`.row-cards`, &pagesCard, chromedp.ByQuery), + ); err != nil { + t.Fatalf("drive analytics page: %v", err) + } + + for _, c := range []struct{ name, got, want string }{ + {"probe requests", probeReq, "5"}, + {"bot requests", botReq, "2"}, + {"notfound requests", notFoundReq, "1"}, + {"total requests (visits only)", totalReq, "2"}, + } { + if strings.TrimSpace(c.got) != c.want { + t.Errorf("%s = %q, want %q", c.name, strings.TrimSpace(c.got), c.want) + } + } + + // The scanner's paths belong in the probe card and nowhere else. + if !strings.Contains(probeCard, "/wp-login.php") { + t.Errorf("probe card doesn't list the probed paths:\n%s", probeCard) + } + if strings.Contains(pagesCard, "/wp-login.php") { + t.Error("a probed path leaked into the visit-facing cards — the kind filter isn't applied") + } + + watch.assertClean(t) +} diff --git a/internal/marketing/templates/privacy.html b/internal/marketing/templates/privacy.html index 91063d6..4d3009b 100644 --- a/internal/marketing/templates/privacy.html +++ b/internal/marketing/templates/privacy.html @@ -72,8 +72,10 @@ the HTTP method, the response status, and a visitor hash. The hash is a salted SHA-256 of the day plus your IP address and user agent — it rotates every day, so the same visitor counts once per day and cannot be followed from one day to the next. - Your IP address is never written down, only mixed into that day's hash. Bot - traffic is skipped. This runs entirely on our own server; no analytics provider is involved. + Your IP address is never written down, only mixed into that day's hash. + Crawlers and vulnerability scanners are recorded the same way but counted separately, so + they don't inflate the visitor figures. This runs entirely on our own server; no analytics + provider is involved.

Diagnostics

diff --git a/internal/store/analytics.go b/internal/store/analytics.go index ccceefe..5173137 100644 --- a/internal/store/analytics.go +++ b/internal/store/analytics.go @@ -16,6 +16,7 @@ type AnalyticsEvent struct { Path string Method string Status int + Kind string // "visit" | "probe" | "notfound" | "bot" — see internal/analytics Visitor string } @@ -26,11 +27,11 @@ func (s *Store) InsertAnalyticsEvents(ctx context.Context, evs []AnalyticsEvent) } rows := make([][]any, len(evs)) for i, e := range evs { - rows[i] = []any{e.Ts, e.Surface, e.Host, e.Path, e.Method, e.Status, e.Visitor} + rows[i] = []any{e.Ts, e.Surface, e.Host, e.Path, e.Method, e.Status, e.Kind, e.Visitor} } _, err := s.pool.CopyFrom(ctx, pgx.Identifier{"analytics_events"}, - []string{"ts", "surface", "host", "path", "method", "status", "visitor"}, + []string{"ts", "surface", "host", "path", "method", "status", "kind", "visitor"}, pgx.CopyFromRows(rows)) if err != nil { return fmt.Errorf("insert analytics events: %w", err) @@ -45,20 +46,20 @@ func (s *Store) RollupAnalytics(ctx context.Context) error { return s.inTx(ctx, func(tx pgx.Tx) error { stmts := []string{ `DELETE FROM analytics_daily WHERE day >= (now() - interval '1 day')::date`, - `INSERT INTO analytics_daily (day, requests, visitors) - SELECT ts::date, count(*), count(DISTINCT visitor) + `INSERT INTO analytics_daily (day, kind, requests, visitors) + SELECT ts::date, kind, count(*), count(DISTINCT visitor) FROM analytics_events WHERE ts >= (now() - interval '1 day')::date - GROUP BY ts::date`, + GROUP BY ts::date, kind`, `DELETE FROM analytics_daily_surface WHERE day >= (now() - interval '1 day')::date`, - `INSERT INTO analytics_daily_surface (day, surface, requests) - SELECT ts::date, surface, count(*) + `INSERT INTO analytics_daily_surface (day, kind, surface, requests) + SELECT ts::date, kind, surface, count(*) FROM analytics_events WHERE ts >= (now() - interval '1 day')::date - GROUP BY ts::date, surface`, + GROUP BY ts::date, kind, surface`, `DELETE FROM analytics_daily_path WHERE day >= (now() - interval '1 day')::date`, - `INSERT INTO analytics_daily_path (day, path, hits) - SELECT ts::date, path, count(*) + `INSERT INTO analytics_daily_path (day, kind, path, hits) + SELECT ts::date, kind, path, count(*) FROM analytics_events WHERE ts >= (now() - interval '1 day')::date - GROUP BY ts::date, path`, + GROUP BY ts::date, kind, path`, } for _, q := range stmts { if _, err := tx.Exec(ctx, q); err != nil { @@ -116,13 +117,39 @@ type VisitorStat struct { Requests int64 } -// AnalyticsTopHosts returns the busiest request hosts over the last `days` days, -// read live from the raw events so the breakdown reflects current traffic. -func (s *Store) AnalyticsTopHosts(ctx context.Context, days, limit int) ([]HostStat, error) { +// KindStat is request volume and distinct sources for one event kind. +type KindStat struct { + Kind string + Requests int64 + Sources int64 +} + +// AnalyticsKindSummary returns per-kind totals over the last `days` days. It +// reads raw events rather than the rollups so "sources" is a true distinct count +// across the window — summing the daily rollup's visitor counts would instead +// count a scanner that runs every day once per day. +func (s *Store) AnalyticsKindSummary(ctx context.Context, days int) ([]KindStat, error) { + rows, err := s.pool.Query(ctx, ` + SELECT kind, count(*), count(DISTINCT visitor) FROM analytics_events + WHERE ts >= now() - ($1::int * interval '1 day') + GROUP BY kind ORDER BY count(*) DESC`, days) + if err != nil { + return nil, fmt.Errorf("analytics kind summary: %w", err) + } + return collectRows(rows, func(r pgx.Row) (KindStat, error) { + var k KindStat + return k, r.Scan(&k.Kind, &k.Requests, &k.Sources) + }) +} + +// AnalyticsTopHosts returns the busiest request hosts of one kind over the last +// `days` days, read live from the raw events so the breakdown reflects current +// traffic. +func (s *Store) AnalyticsTopHosts(ctx context.Context, days int, kind string, limit int) ([]HostStat, error) { rows, err := s.pool.Query(ctx, ` SELECT host, count(*) FROM analytics_events - WHERE ts >= now() - ($1::int * interval '1 day') - GROUP BY host ORDER BY count(*) DESC LIMIT $2`, days, limit) + WHERE ts >= now() - ($1::int * interval '1 day') AND kind = $2 + GROUP BY host ORDER BY count(*) DESC LIMIT $3`, days, kind, limit) if err != nil { return nil, fmt.Errorf("analytics top hosts: %w", err) } @@ -132,18 +159,20 @@ func (s *Store) AnalyticsTopHosts(ctx context.Context, days, limit int) ([]HostS }) } -// AnalyticsTopVisitors returns the busiest visitor hashes over the last `days` -// days, with the surface and most-recent path each was seen on. Hashes rotate -// daily, so a heavy daily user appears once per day, not once overall. -func (s *Store) AnalyticsTopVisitors(ctx context.Context, days, limit int) ([]VisitorStat, error) { +// AnalyticsTopVisitors returns the busiest visitor hashes of one kind over the +// last `days` days, with the surface and most-recent path each was seen on. +// Hashes rotate daily, so a heavy daily user appears once per day, not once +// overall. Filtering by kind is what keeps a scanner replaying a wordlist out of +// the visitor list — it can outweigh every real reader combined. +func (s *Store) AnalyticsTopVisitors(ctx context.Context, days int, kind string, limit int) ([]VisitorStat, error) { rows, err := s.pool.Query(ctx, ` SELECT e.visitor, count(*) AS n, (array_agg(e.surface ORDER BY e.ts DESC))[1] AS surface, max(e.ts) AS last_seen, (array_agg(e.path ORDER BY e.ts DESC))[1] AS last_path FROM analytics_events e - WHERE e.ts >= now() - ($1::int * interval '1 day') - GROUP BY e.visitor ORDER BY n DESC LIMIT $2`, days, limit) + WHERE e.ts >= now() - ($1::int * interval '1 day') AND e.kind = $2 + GROUP BY e.visitor ORDER BY n DESC LIMIT $3`, days, kind, limit) if err != nil { return nil, fmt.Errorf("analytics top visitors: %w", err) } @@ -153,11 +182,12 @@ func (s *Store) AnalyticsTopVisitors(ctx context.Context, days, limit int) ([]Vi }) } -// AnalyticsDaily returns the last `days` days of totals, oldest first. -func (s *Store) AnalyticsDaily(ctx context.Context, days int) ([]DayStat, error) { +// AnalyticsDaily returns the last `days` days of totals for one kind, oldest +// first. +func (s *Store) AnalyticsDaily(ctx context.Context, days int, kind string) ([]DayStat, error) { rows, err := s.pool.Query(ctx, ` SELECT day, requests, visitors FROM analytics_daily - WHERE day >= (now()::date - $1::int) ORDER BY day`, days) + WHERE day >= (now()::date - $1::int) AND kind = $2 ORDER BY day`, days, kind) if err != nil { return nil, fmt.Errorf("analytics daily: %w", err) } @@ -167,11 +197,13 @@ func (s *Store) AnalyticsDaily(ctx context.Context, days int) ([]DayStat, error) }) } -// AnalyticsBySurface returns request volume per surface over the last `days` days. -func (s *Store) AnalyticsBySurface(ctx context.Context, days int) ([]SurfaceStat, error) { +// AnalyticsBySurface returns request volume per surface for one kind over the +// last `days` days. +func (s *Store) AnalyticsBySurface(ctx context.Context, days int, kind string) ([]SurfaceStat, error) { rows, err := s.pool.Query(ctx, ` SELECT surface, sum(requests) FROM analytics_daily_surface - WHERE day >= (now()::date - $1::int) GROUP BY surface ORDER BY sum(requests) DESC`, days) + WHERE day >= (now()::date - $1::int) AND kind = $2 + GROUP BY surface ORDER BY sum(requests) DESC`, days, kind) if err != nil { return nil, fmt.Errorf("analytics by surface: %w", err) } @@ -181,11 +213,14 @@ func (s *Store) AnalyticsBySurface(ctx context.Context, days int) ([]SurfaceStat }) } -// AnalyticsTopPaths returns the busiest paths over the last `days` days. -func (s *Store) AnalyticsTopPaths(ctx context.Context, days, limit int) ([]PathStat, error) { +// AnalyticsTopPaths returns the busiest paths of one kind over the last `days` +// days — the pages people read for "visit", the wordlist being replayed for +// "probe". +func (s *Store) AnalyticsTopPaths(ctx context.Context, days int, kind string, limit int) ([]PathStat, error) { rows, err := s.pool.Query(ctx, ` SELECT path, sum(hits) FROM analytics_daily_path - WHERE day >= (now()::date - $1::int) GROUP BY path ORDER BY sum(hits) DESC LIMIT $2`, days, limit) + WHERE day >= (now()::date - $1::int) AND kind = $2 + GROUP BY path ORDER BY sum(hits) DESC LIMIT $3`, days, kind, limit) if err != nil { return nil, fmt.Errorf("analytics top paths: %w", err) } diff --git a/internal/store/analytics_test.go b/internal/store/analytics_test.go index f6cae55..200ce1d 100644 --- a/internal/store/analytics_test.go +++ b/internal/store/analytics_test.go @@ -11,9 +11,9 @@ func TestAnalyticsRollup(t *testing.T) { now := time.Now() evs := []AnalyticsEvent{ - {Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Visitor: "v1"}, - {Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Visitor: "v1"}, - {Ts: now, Surface: "app", Host: "h", Path: "/b", Method: "GET", Status: 200, Visitor: "v2"}, + {Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Kind: "visit", Visitor: "v1"}, + {Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Kind: "visit", Visitor: "v1"}, + {Ts: now, Surface: "app", Host: "h", Path: "/b", Method: "GET", Status: 200, Kind: "visit", Visitor: "v2"}, } if err := st.InsertAnalyticsEvents(ctx, evs); err != nil { t.Fatalf("insert: %v", err) @@ -22,7 +22,7 @@ func TestAnalyticsRollup(t *testing.T) { t.Fatalf("rollup: %v", err) } - daily, err := st.AnalyticsDaily(ctx, 2) + daily, err := st.AnalyticsDaily(ctx, 2, "visit") if err != nil { t.Fatalf("daily: %v", err) } @@ -30,7 +30,7 @@ func TestAnalyticsRollup(t *testing.T) { t.Fatalf("daily = %+v, want one day with 3 requests / 2 visitors", daily) } - surf, err := st.AnalyticsBySurface(ctx, 2) + surf, err := st.AnalyticsBySurface(ctx, 2, "visit") if err != nil { t.Fatalf("surface: %v", err) } @@ -42,7 +42,7 @@ func TestAnalyticsRollup(t *testing.T) { t.Fatalf("by surface = %v, want root=2 app=1", bySurface) } - paths, err := st.AnalyticsTopPaths(ctx, 2, 10) + paths, err := st.AnalyticsTopPaths(ctx, 2, "visit", 10) if err != nil { t.Fatalf("paths: %v", err) } @@ -54,42 +54,180 @@ func TestAnalyticsRollup(t *testing.T) { if err := st.RollupAnalytics(ctx); err != nil { t.Fatalf("rollup again: %v", err) } - daily, _ = st.AnalyticsDaily(ctx, 2) + daily, _ = st.AnalyticsDaily(ctx, 2, "visit") if len(daily) != 1 || daily[0].Requests != 3 { t.Fatalf("after re-rollup daily = %+v, want unchanged", daily) } } +// TestAnalyticsRollupSeparatesKinds is the regression this whole column exists +// for: one scanner replaying a wordlist used to outweigh every real visitor in +// the rollups. Its requests must still be stored and countable, just never mixed +// into the visit figures. +func TestAnalyticsRollupSeparatesKinds(t *testing.T) { + t.Parallel() + st, ctx := orgTestStore(t) + now := time.Now() + + evs := []AnalyticsEvent{ + {Ts: now, Surface: "root", Host: "h", Path: "/", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"}, + {Ts: now, Surface: "root", Host: "h", Path: "/docs", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"}, + {Ts: now, Surface: "root", Host: "h", Path: "/orgs", Method: "GET", Status: 200, Kind: "bot", Visitor: "googlebot"}, + {Ts: now, Surface: "root", Host: "h", Path: "/gone", Method: "GET", Status: 404, Kind: "notfound", Visitor: "alice"}, + } + // One scanner, many paths, all in a day — the shape that swamped the charts. + for i, p := range []string{"/wp-login.php", "/.env", "/index.php", "/firebase-key.json", "/manager/html"} { + evs = append(evs, AnalyticsEvent{ + Ts: now.Add(-time.Duration(i) * time.Second), Surface: "root", Host: "h", + Path: p, Method: "GET", Status: 404, Kind: "probe", Visitor: "scanner", + }) + } + if err := st.InsertAnalyticsEvents(ctx, evs); err != nil { + t.Fatalf("insert: %v", err) + } + if err := st.RollupAnalytics(ctx); err != nil { + t.Fatalf("rollup: %v", err) + } + + // Visits are unpolluted: 2 requests from 1 visitor, not 9 from 4. + visits, err := st.AnalyticsDaily(ctx, 2, "visit") + if err != nil { + t.Fatalf("daily visits: %v", err) + } + if len(visits) != 1 || visits[0].Requests != 2 || visits[0].Visitors != 1 { + t.Fatalf("visit day = %+v, want 2 requests / 1 visitor", visits) + } + + // The scanner is still fully accounted for under its own kind. + probes, err := st.AnalyticsDaily(ctx, 2, "probe") + if err != nil { + t.Fatalf("daily probes: %v", err) + } + if len(probes) != 1 || probes[0].Requests != 5 || probes[0].Visitors != 1 { + t.Fatalf("probe day = %+v, want 5 requests / 1 source", probes) + } + + bots, err := st.AnalyticsDaily(ctx, 2, "bot") + if err != nil { + t.Fatalf("daily bots: %v", err) + } + if len(bots) != 1 || bots[0].Requests != 1 { + t.Fatalf("bot day = %+v, want 1 request", bots) + } + + // Per-kind path and visitor views read only their own kind. + probePaths, err := st.AnalyticsTopPaths(ctx, 2, "probe", 10) + if err != nil { + t.Fatalf("probe paths: %v", err) + } + if len(probePaths) != 5 { + t.Fatalf("probe paths = %+v, want the 5 probed paths", probePaths) + } + visitPaths, err := st.AnalyticsTopPaths(ctx, 2, "visit", 10) + if err != nil { + t.Fatalf("visit paths: %v", err) + } + for _, p := range visitPaths { + if p.Path == "/wp-login.php" { + t.Fatalf("a probed path leaked into the visit paths: %+v", visitPaths) + } + } + + vis, err := st.AnalyticsTopVisitors(ctx, 2, "visit", 10) + if err != nil { + t.Fatalf("visitors: %v", err) + } + if len(vis) != 1 || vis[0].Visitor != "alice" { + t.Fatalf("top visitors = %+v, want only alice — the scanner must not rank here", vis) + } +} + +// TestAnalyticsKindSummary: the cards need a true distinct-source count across +// the window. Summing the daily rollups would count a scanner that runs every +// day once per day, turning one attacker into a crowd. +func TestAnalyticsKindSummary(t *testing.T) { + t.Parallel() + st, ctx := orgTestStore(t) + now := time.Now() + + evs := []AnalyticsEvent{ + {Ts: now, Surface: "root", Host: "h", Path: "/", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"}, + {Ts: now, Surface: "root", Host: "h", Path: "/docs", Method: "GET", Status: 200, Kind: "visit", Visitor: "bob"}, + {Ts: now, Surface: "root", Host: "h", Path: "/orgs", Method: "GET", Status: 200, Kind: "bot", Visitor: "googlebot"}, + {Ts: now, Surface: "root", Host: "h", Path: "/gone", Method: "GET", Status: 404, Kind: "notfound", Visitor: "alice"}, + } + // The same scanner across two days: 4 requests, but one source. + for i, p := range []string{"/.env", "/wp-login.php", "/.git/config", "/index.php"} { + evs = append(evs, AnalyticsEvent{ + Ts: now.Add(-time.Duration(i) * 12 * time.Hour), Surface: "root", Host: "h", + Path: p, Method: "GET", Status: 404, Kind: "probe", Visitor: "scanner", + }) + } + if err := st.InsertAnalyticsEvents(ctx, evs); err != nil { + t.Fatalf("insert: %v", err) + } + + kinds, err := st.AnalyticsKindSummary(ctx, 7) + if err != nil { + t.Fatalf("kind summary: %v", err) + } + got := map[string]KindStat{} + for _, k := range kinds { + got[k.Kind] = k + } + if p := got["probe"]; p.Requests != 4 || p.Sources != 1 { + t.Errorf("probe = %+v, want 4 requests from 1 source", p) + } + if v := got["visit"]; v.Requests != 2 || v.Sources != 2 { + t.Errorf("visit = %+v, want 2 requests from 2 sources", v) + } + if b := got["bot"]; b.Requests != 1 || b.Sources != 1 { + t.Errorf("bot = %+v, want 1 request from 1 source", b) + } + if n := got["notfound"]; n.Requests != 1 { + t.Errorf("notfound = %+v, want 1 request", n) + } +} + func TestAnalyticsHostsAndVisitors(t *testing.T) { t.Parallel() st, ctx := orgTestStore(t) now := time.Now() evs := []AnalyticsEvent{ - {Ts: now.Add(-time.Minute), Surface: "custom", Host: "1.2.3.4", Path: "/", Method: "GET", Status: 200, Visitor: "bot"}, - {Ts: now, Surface: "custom", Host: "1.2.3.4", Path: "/robots.txt", Method: "GET", Status: 200, Visitor: "bot"}, - {Ts: now, Surface: "app", Host: "app.x", Path: "/dash", Method: "GET", Status: 200, Visitor: "alice"}, + {Ts: now.Add(-time.Minute), Surface: "custom", Host: "1.2.3.4", Path: "/", Method: "GET", Status: 200, Kind: "bot", Visitor: "bot"}, + {Ts: now, Surface: "custom", Host: "1.2.3.4", Path: "/robots.txt", Method: "GET", Status: 200, Kind: "bot", Visitor: "bot"}, + {Ts: now, Surface: "app", Host: "app.x", Path: "/dash", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"}, } if err := st.InsertAnalyticsEvents(ctx, evs); err != nil { t.Fatalf("insert: %v", err) } - hosts, err := st.AnalyticsTopHosts(ctx, 2, 10) + hosts, err := st.AnalyticsTopHosts(ctx, 2, "bot", 10) if err != nil { t.Fatalf("hosts: %v", err) } - if len(hosts) != 2 || hosts[0].Host != "1.2.3.4" || hosts[0].Requests != 2 { - t.Fatalf("top hosts = %+v, want 1.2.3.4=2 first", hosts) + if len(hosts) != 1 || hosts[0].Host != "1.2.3.4" || hosts[0].Requests != 2 { + t.Fatalf("top bot hosts = %+v, want 1.2.3.4=2", hosts) } - vis, err := st.AnalyticsTopVisitors(ctx, 2, 10) + vis, err := st.AnalyticsTopVisitors(ctx, 2, "bot", 10) if err != nil { t.Fatalf("visitors: %v", err) } - if len(vis) != 2 || vis[0].Visitor != "bot" || vis[0].Requests != 2 { - t.Fatalf("top visitors = %+v, want bot=2 first", vis) + if len(vis) != 1 || vis[0].Visitor != "bot" || vis[0].Requests != 2 { + t.Fatalf("top bot visitors = %+v, want bot=2", vis) } if vis[0].Surface != "custom" || vis[0].LastPath != "/robots.txt" { t.Fatalf("bot visitor surface/last-path = %q/%q, want custom//robots.txt", vis[0].Surface, vis[0].LastPath) } + + // The human is reachable under her own kind, and only there. + human, err := st.AnalyticsTopVisitors(ctx, 2, "visit", 10) + if err != nil { + t.Fatalf("visit visitors: %v", err) + } + if len(human) != 1 || human[0].Visitor != "alice" { + t.Fatalf("visit visitors = %+v, want only alice", human) + } } diff --git a/internal/store/migrations/0045_analytics_kind.sql b/internal/store/migrations/0045_analytics_kind.sql new file mode 100644 index 0000000..68a5390 --- /dev/null +++ b/internal/store/migrations/0045_analytics_kind.sql @@ -0,0 +1,95 @@ +-- +goose Up +-- Classify each recorded request so scanner noise stops being counted as +-- traffic. A single vulnerability scanner can post hundreds of 404s a day under +-- a browser user agent, which previously landed it at the top of "top visitors" +-- and buried the real ones. kind is decided at record time (internal/analytics +-- classify.go): "visit" | "probe" | "notfound" | "bot". +ALTER TABLE analytics_events ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit'; +CREATE INDEX analytics_events_kind_ts_idx ON analytics_events (kind, ts); + +-- The rollups gain the same dimension so each kind keeps its own daily history +-- and the admin charts can read one kind without scanning raw events. +ALTER TABLE analytics_daily DROP CONSTRAINT analytics_daily_pkey; +ALTER TABLE analytics_daily ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit'; +ALTER TABLE analytics_daily ADD PRIMARY KEY (day, kind); + +ALTER TABLE analytics_daily_surface DROP CONSTRAINT analytics_daily_surface_pkey; +ALTER TABLE analytics_daily_surface ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit'; +ALTER TABLE analytics_daily_surface ADD PRIMARY KEY (day, kind, surface); + +ALTER TABLE analytics_daily_path DROP CONSTRAINT analytics_daily_path_pkey; +ALTER TABLE analytics_daily_path ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit'; +ALTER TABLE analytics_daily_path ADD PRIMARY KEY (day, kind, path); + +-- Backfill history with the same rules the Go classifier applies, so the +-- existing retention window isn't left misreporting scanners as visitors. Bots +-- can't be backfilled — they were dropped before insert and never stored, and +-- no user agent is retained to re-derive them. +-- The gate is "not a 2xx", not "is a 404": a scanner sweeping the www host gets +-- a 301 to the apex, and gating on 404 alone left all of that counted as real +-- traffic. `.json` is absent from the extension list and generic names are +-- matched as whole segments only — /orgs/{id}/repeaters.json, +-- /repeaters/{id}/config.json and /repeaters/{id}/console are real routes. +UPDATE analytics_events SET kind = CASE + WHEN status NOT BETWEEN 200 AND 299 AND ( + -- an unfilled placeholder from the scanner's own template + path LIKE '%*%' + -- generic names that are only suspicious at the root of a host + OR lower(path) IN ('/api', '/info', '/env', '/server', '/phpinfo', '/console', '/console/', + '/config.json', '/config.js', '/aws.config.js', + '/server-status', '/server-info', '/v2/_catalog', '/old/') + -- file types we serve nowhere, in ANY segment: the Laravel Ignition RCE + -- arrives as /index.php/_ignition/... with the .php mid-path. Editor and + -- backup droppings may follow the real extension (/phpinfo.php.save, + -- /phpinfo.php~), so they're allowed to trail the match. + OR path ~* '\.(php|phps|php[357]|ini|env|ya?ml|sql|py|tfstate|properties|js|aspx?|axd|cgi|jspx?|action|bak|old|swp|save|orig|copy|dist|tmp)(~|\.(save|bak|old|orig|copy|dist|tmp|backup))*($|/)' + -- other stacks' credential and config files, as whole segments + OR path ~* '/(firebase-key\.json|credentials\.json|service-account\.json|secrets\.json|settings\.json|appsettings\.json|sftp\.json|package\.json|composer\.json|web\.config|id_rsa|id_dsa|dockerfile|backup\.(zip|tar\.gz))($|/)' + -- JSON outside the two prefixes where we actually serve it + OR (path ~* '\.json($|/)' AND path !~ '^/(orgs|repeaters)/') + -- scanner wordlist fragments: other stacks' panels and RCE entry points + OR path ~* '(wp-|wordpress|xmlrpc|phpmyadmin|/pma/|adminer|cgi-bin|/vendor/|autodiscover|/owa/|/ecp/|manager/html|/solr/|jenkins|actuator|telescope|eval-stdin|hnap1|graphql|/gql|_profiler|@vite|___proxy_subdomain|debug/default|_catalog|_ignition|webhook-waiting|stats/prometheus|/goform/|/boaform/|_environment|meta-inf)' + -- we serve no dotfiles; .well-known is the one real convention + OR (path ~ '/\.[^/]' AND path !~* '/\.well-known($|/)') + ) THEN 'probe' + WHEN status = 404 THEN 'notfound' + ELSE 'visit' +END; + +-- Rebuild every rollup from the reclassified raw events. Raw retention (90d) +-- covers the whole window the dashboard can display, so this is a complete +-- rebuild rather than a partial correction. +DELETE FROM analytics_daily; +INSERT INTO analytics_daily (day, kind, requests, visitors) +SELECT ts::date, kind, count(*), count(DISTINCT visitor) FROM analytics_events +GROUP BY ts::date, kind; + +DELETE FROM analytics_daily_surface; +INSERT INTO analytics_daily_surface (day, kind, surface, requests) +SELECT ts::date, kind, surface, count(*) FROM analytics_events +GROUP BY ts::date, kind, surface; + +DELETE FROM analytics_daily_path; +INSERT INTO analytics_daily_path (day, kind, path, hits) +SELECT ts::date, kind, path, count(*) FROM analytics_events +GROUP BY ts::date, kind, path; + +-- +goose Down +DELETE FROM analytics_daily WHERE kind <> 'visit'; +DELETE FROM analytics_daily_surface WHERE kind <> 'visit'; +DELETE FROM analytics_daily_path WHERE kind <> 'visit'; + +ALTER TABLE analytics_daily_path DROP CONSTRAINT analytics_daily_path_pkey; +ALTER TABLE analytics_daily_path DROP COLUMN kind; +ALTER TABLE analytics_daily_path ADD PRIMARY KEY (day, path); + +ALTER TABLE analytics_daily_surface DROP CONSTRAINT analytics_daily_surface_pkey; +ALTER TABLE analytics_daily_surface DROP COLUMN kind; +ALTER TABLE analytics_daily_surface ADD PRIMARY KEY (day, surface); + +ALTER TABLE analytics_daily DROP CONSTRAINT analytics_daily_pkey; +ALTER TABLE analytics_daily DROP COLUMN kind; +ALTER TABLE analytics_daily ADD PRIMARY KEY (day); + +DROP INDEX analytics_events_kind_ts_idx; +ALTER TABLE analytics_events DROP COLUMN kind;