Split out probe traffic in analytics page

internal/analytics/analytics.go

@@ -132,13 +132,15 @@ func (rec *Recorder) Handler(next http.Handler) http.Handler {
 // buffer is full the event is dropped rather than slowing the response.
 func (rec *Recorder) record(r *http.Request, status int) {
 	host := web.HostWithoutPort(r.Host)
+	path := web.RedactPath(r.URL.Path)
 	ev := store.AnalyticsEvent{
 		Ts:      time.Now(),
 		Surface: rec.surface(host),
 		Host:    host,
-		Path:    web.RedactPath(r.URL.Path),
+		Path:    path,
 		Method:  r.Method,
 		Status:  status,
+		Kind:    classify(path, status, r.UserAgent()),
 		Visitor: rec.visitor(r),
 	}
 	select {
@@ -174,8 +176,10 @@ func (rec *Recorder) visitor(r *http.Request) string {
 	return hex.EncodeToString(h.Sum(nil)[:8])
 }
 
-// skip drops health checks, static assets, websockets, preflight/HEAD, and bots —
-// "people visiting", not infrastructure noise.
+// skip drops health checks, static assets, websockets, and preflight/HEAD —
+// infrastructure that isn't a request for a page. Bots and scanners are no
+// longer dropped here: they're recorded under their own kind (see classify) so
+// their volume is visible without being counted as visits.
 func skip(r *http.Request) bool {
 	if r.Method == http.MethodOptions || r.Method == http.MethodHead {
 		return true
@@ -192,19 +196,6 @@ func skip(r *http.Request) bool {
 		strings.HasSuffix(p, "/ws"):
 		return true
 	}
-	return isBot(r.UserAgent())
-}
-
-func isBot(ua string) bool {
-	ua = strings.ToLower(ua)
-	if ua == "" {
-		return true
-	}
-	for _, s := range []string{"bot", "crawl", "spider", "slurp", "headless", "preview", "monitor"} {
-		if strings.Contains(ua, s) {
-			return true
-		}
-	}
 	return false
 }
 

internal/analytics/analytics_test.go

@@ -61,8 +61,6 @@ func TestHandlerSkips(t *testing.T) {
 		{"/healthz", "Mozilla/5.0"},
 		{"/static/app.css", "Mozilla/5.0"},
 		{"/repeaters/abc/ws", "Mozilla/5.0"},
-		{"/dashboard", "Googlebot/2.1"}, // bot UA
-		{"/dashboard", ""},              // empty UA
 	}
 	for _, c := range cases {
 		req := httptest.NewRequest(http.MethodGet, "http://app.x"+c.path, nil)
@@ -78,6 +76,70 @@ func TestHandlerSkips(t *testing.T) {
 	}
 }
 
+// TestHandlerRecordsKind covers the whole point of the kind column: scanners and
+// crawlers reach the store rather than being dropped at the door, but under a
+// kind the dashboard can hold apart from real visits.
+func TestHandlerRecordsKind(t *testing.T) {
+	cases := []struct {
+		name, path, ua string
+		status         int
+		want           string
+	}{
+		{"visit", "/dashboard", "Mozilla/5.0", http.StatusOK, KindVisit},
+		{"probe", "/wp-login.php", "Mozilla/5.0", http.StatusNotFound, KindProbe},
+		{"bot", "/orgs", "Googlebot/2.1", http.StatusOK, KindBot},
+		{"empty ua is a bot", "/orgs", "", http.StatusOK, KindBot},
+		{"broken link", "/orgs/gone", "Mozilla/5.0", http.StatusNotFound, KindNotFound},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			rec := testRecorder()
+			h := rec.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(c.status)
+			}))
+			req := httptest.NewRequest(http.MethodGet, "http://app.x"+c.path, nil)
+			if c.ua != "" {
+				req.Header.Set("User-Agent", c.ua)
+			}
+			h.ServeHTTP(httptest.NewRecorder(), req)
+
+			select {
+			case e := <-rec.ch:
+				if e.Kind != c.want {
+					t.Fatalf("kind = %q, want %q (event %+v)", e.Kind, c.want, e)
+				}
+			default:
+				t.Fatalf("%s was dropped; it should be recorded as %q", c.name, c.want)
+			}
+		})
+	}
+}
+
+// TestKindUsesRedactedPath: classification runs on the path as stored, so a
+// secret in the URL can't reach the classifier's signature lists either.
+func TestKindUsesRedactedPath(t *testing.T) {
+	rec := testRecorder()
+	h := rec.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "nope", http.StatusNotFound)
+	}))
+
+	req := httptest.NewRequest(http.MethodGet, "http://app.x/invite/S3cr3tShareToken.php", nil)
+	req.Header.Set("User-Agent", "Mozilla/5.0")
+	h.ServeHTTP(httptest.NewRecorder(), req)
+
+	select {
+	case e := <-rec.ch:
+		if e.Path != "/invite/:token" {
+			t.Fatalf("recorded path = %q, want the token redacted", e.Path)
+		}
+		if e.Kind != KindNotFound {
+			t.Fatalf("kind = %q, want %q — the .php was in the redacted-away token", e.Kind, KindNotFound)
+		}
+	default:
+		t.Fatal("expected an event to be enqueued")
+	}
+}
+
 func TestSurfaceClassification(t *testing.T) {
 	rec := testRecorder()
 	for host, want := range map[string]string{

internal/analytics/classify.go

@@ -0,0 +1,190 @@
+package analytics
+
+import "strings"
+
+// Event kinds. Every recorded request gets exactly one, decided at record time
+// by classify. They're stored on the row and on every rollup so the dashboard
+// can read one kind at a time — scanner noise is kept, not discarded, but it
+// never lands in the same bucket as a person visiting a page.
+const (
+	KindVisit    = "visit"    // a real request that hit a real route
+	KindProbe    = "probe"    // a request for something only an attacker asks for
+	KindNotFound = "notfound" // a 404 with no attack signature — likely a broken link
+	KindBot      = "bot"      // a self-identified crawler or monitor
+)
+
+// probeExtensions are file types we serve nowhere. Matched against every path
+// SEGMENT, not just the last one: the Laravel Ignition RCE arrives as
+// /index.php/_ignition/execute-solution, where the ".php" sits mid-path.
+var probeExtensions = []string{
+	".php", ".phps", ".php3", ".php5", ".php7",
+	".ini", ".env", ".yaml", ".yml", ".sql", ".py", ".tfstate", ".properties",
+	".asp", ".aspx", ".axd", ".cgi", ".jsp", ".jspx", ".action",
+	// .js is safe to claim: the only JavaScript we serve lives under /static/,
+	// which skip() drops before anything is recorded, so a .js that reaches the
+	// classifier is by definition not ours.
+	".js",
+	// Editor and backup droppings. These also arrive appended to a real
+	// extension (/phpinfo.php.save, /config.json.save), which is why the check
+	// below strips them and re-tests rather than only matching the tail.
+	".bak", ".old", ".swp", ".save", ".orig", ".copy", ".dist", ".tmp",
+}
+
+// backupSuffixes get stripped from a segment before the extension check runs, so
+// /phpinfo.php~ is recognized as the .php probe it is.
+var backupSuffixes = []string{"~", ".save", ".bak", ".old", ".orig", ".copy", ".dist", ".tmp", ".backup"}
+
+// probeNames match a path segment exactly — credential and config files that
+// ship with other stacks. Matching the whole segment (never a substring) is what
+// keeps our own /orgs/{id}/repeaters.json and /repeaters/{id}/config.json out of
+// this bucket; note that bare "config.json" is deliberately absent for exactly
+// that reason and is handled by rootProbes instead.
+var probeNames = []string{
+	"firebase-key.json", "credentials.json", "service-account.json",
+	"secrets.json", "settings.json", "appsettings.json", "sftp.json",
+	"package.json", "composer.json", "web.config",
+	"id_rsa", "id_dsa", "backup.zip", "backup.tar.gz", "dockerfile",
+}
+
+// jsonRoots are the only path prefixes under which we serve JSON
+// (/orgs/{id}/repeaters.json and /repeaters/{id}/config.json). A .json anywhere
+// else is someone fishing for another stack's credentials — production alone
+// turned up gcp-credentials.json, firebase-adminsdk.json, aws-ses.json and
+// appsettings.Production.json, which no fixed list of names would have kept up
+// with. Scoping by prefix rather than by name keeps a 404 on our own two
+// endpoints readable as the broken link it is.
+var jsonRoots = []string{"/orgs/", "/repeaters/"}
+
+// rootProbes are generic names that are only suspicious at the root of a host.
+// "/api" and "/console" are scanner bait; /api/login/begin and
+// /repeaters/{id}/console are ours. Exact full-path matches only.
+var rootProbes = []string{
+	"/api", "/info", "/env", "/server", "/phpinfo", "/console", "/console/",
+	"/config.json", "/config.js", "/aws.config.js",
+	"/server-status", "/server-info", "/v2/_catalog", "/old/",
+}
+
+// probeSegments are fragments from the standard scanner wordlists — other
+// stacks' admin panels, framework internals, and known RCE entry points. Each is
+// specific enough not to collide with our own URL space.
+var probeSegments = []string{
+	"wp-", "wordpress", "xmlrpc", "phpmyadmin", "/pma/", "adminer", "cgi-bin",
+	"/vendor/", "autodiscover", "/owa/", "/ecp/", "manager/html", "/solr/",
+	"jenkins", "actuator", "telescope", "eval-stdin", "hnap1",
+	"graphql", "/gql", "_profiler", "@vite", "___proxy_subdomain",
+	"debug/default", "_catalog", "_ignition", "webhook-waiting",
+	"stats/prometheus", "/goform/", "/boaform/", "_environment", "meta-inf",
+}
+
+// classify buckets one finished request.
+//
+// A probe is a request for something we don't serve, that only an attacker asks
+// for. The status gate is "not a 2xx": a scanner sweeping the www host gets a
+// 301 to the apex rather than a 404, and gating on 404 alone let all of that
+// through as ordinary traffic. A 2xx means we really do serve the path, so the
+// signature must be wrong and the request stays a visit — that direction is the
+// safe one to be wrong in.
+//
+// The signatures only decide which flavor of non-2xx it was, so a miss can never
+// hide a real request: it degrades to "notfound", still visible, just not
+// attributed to an attacker. Bots are checked after probes because a scanner is
+// free to put "bot" in its user agent, and what it asked for is better evidence
+// than what it calls itself.
+//
+// The SQL backfill in migration 0045 mirrors these rules; it runs once over
+// history and the two aren't kept in lockstep afterwards.
+func classify(path string, status int, ua string) string {
+	if !isSuccess(status) && isProbePath(path) {
+		return KindProbe
+	}
+	if isBot(ua) {
+		return KindBot
+	}
+	if status == 404 {
+		return KindNotFound
+	}
+	return KindVisit
+}
+
+// isSuccess reports whether the response actually served the path.
+func isSuccess(status int) bool { return status >= 200 && status < 300 }
+
+// isProbePath reports whether a path looks like it came off a scanner wordlist.
+// Case-insensitive: the same list gets replayed in every casing.
+func isProbePath(path string) bool {
+	p := strings.ToLower(path)
+
+	// A literal "*" is an unfilled placeholder from the scanner's own template
+	// (/workspaces/*, /webhook-waiting/*). No browser ever sends one.
+	if strings.Contains(p, "*") {
+		return true
+	}
+	for _, s := range rootProbes {
+		if p == s {
+			return true
+		}
+	}
+	for _, s := range probeSegments {
+		if strings.Contains(p, s) {
+			return true
+		}
+	}
+	jsonIsOurs := false
+	for _, root := range jsonRoots {
+		if strings.HasPrefix(p, root) {
+			jsonIsOurs = true
+			break
+		}
+	}
+
+	for _, seg := range strings.Split(p, "/") {
+		if seg == "" {
+			continue
+		}
+		// We serve no dotfiles. .well-known is the one real convention, and
+		// exempting it keeps security.txt and friends out of the attack bucket.
+		if seg[0] == '.' && seg != ".well-known" {
+			return true
+		}
+		if strings.HasSuffix(seg, ".json") && !jsonIsOurs {
+			return true
+		}
+		for _, s := range probeNames {
+			if seg == s {
+				return true
+			}
+		}
+		// Strip editor/backup droppings before testing the extension, so
+		// /phpinfo.php.save and /phpinfo.php~ read as the .php probes they are.
+		base := seg
+		for changed := true; changed; {
+			changed = false
+			for _, s := range backupSuffixes {
+				if trimmed, ok := strings.CutSuffix(base, s); ok && trimmed != "" {
+					base, changed = trimmed, true
+				}
+			}
+		}
+		for _, s := range probeExtensions {
+			if strings.HasSuffix(seg, s) || strings.HasSuffix(base, s) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// isBot reports whether the user agent identifies itself as automated. An empty
+// user agent counts: every real browser sends one.
+func isBot(ua string) bool {
+	ua = strings.ToLower(ua)
+	if ua == "" {
+		return true
+	}
+	for _, s := range []string{"bot", "crawl", "spider", "slurp", "headless", "preview", "monitor"} {
+		if strings.Contains(ua, s) {
+			return true
+		}
+	}
+	return false
+}

internal/analytics/classify_test.go

@@ -0,0 +1,187 @@
+package analytics
+
+import "testing"
+
+const browserUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
+
+func TestClassify(t *testing.T) {
+	cases := []struct {
+		name   string
+		path   string
+		status int
+		ua     string
+		want   string
+	}{
+		// Real traffic.
+		{"page view", "/orgs/example-mesh", 200, browserUA, KindVisit},
+		{"redirect", "/", 301, browserUA, KindVisit},
+		{"form post", "/orgs/new", 302, browserUA, KindVisit},
+		{"server error is still a visit", "/dashboard", 500, browserUA, KindVisit},
+		{"forbidden is still a visit", "/admin/users", 403, browserUA, KindVisit},
+
+		// Scanner traffic, all pulled from production logs. Every one arrived with
+		// a browser user agent, which is why the UA check alone never caught them.
+		{"php config", "/wp_mail_smtp.ini", 404, browserUA, KindProbe},
+		{"php index", "/index.php", 404, browserUA, KindProbe},
+		{"phpinfo", "/includes/phpinfo.php", 404, browserUA, KindProbe},
+		{"firebase key", "/firebase-key.json", 404, browserUA, KindProbe},
+		{"cli config", "/.vultr-cli.yaml", 404, browserUA, KindProbe},
+		{"tomcat manager", "/manager/html", 404, browserUA, KindProbe},
+		{"dotenv", "/.env", 404, browserUA, KindProbe},
+		{"git config", "/.git/config", 404, browserUA, KindProbe},
+		{"wordpress login", "/wp-login.php", 404, browserUA, KindProbe},
+		{"uppercase replay", "/WP-ADMIN/SETUP-CONFIG.PHP", 404, browserUA, KindProbe},
+
+		// Missed by the first version of this list — the reason it was rewritten.
+		{"laravel ignition rce", "/index.php/_ignition/execute-solution", 404, browserUA, KindProbe},
+		{"graphql", "/graphql", 404, browserUA, KindProbe},
+		{"graphql under api", "/api/graphql", 404, browserUA, KindProbe},
+		{"apache status", "/server-status", 404, browserUA, KindProbe},
+		{"docker registry", "/v2/_catalog", 404, browserUA, KindProbe},
+		{"vite env", "/@vite/env", 404, browserUA, KindProbe},
+		{"aspnet trace", "/trace.axd", 404, browserUA, KindProbe},
+		{"struts action", "/login.action", 404, browserUA, KindProbe},
+		{"cpanel proxy", "/___proxy_subdomain_cpanel", 404, browserUA, KindProbe},
+		{"symfony profiler", "/_profiler/phpinfo", 404, browserUA, KindProbe},
+		{"ds_store", "/.DS_Store", 404, browserUA, KindProbe},
+		{"stripe dotfile", "/.stripe/", 404, browserUA, KindProbe},
+		{"bare api", "/api", 404, browserUA, KindProbe},
+		{"bare console", "/console/", 404, browserUA, KindProbe},
+		{"root config.json", "/config.json", 404, browserUA, KindProbe},
+		{"unfilled wildcard", "/workspaces/*", 404, browserUA, KindProbe},
+		{"unfilled wildcard 2", "/webhook-waiting/*", 404, browserUA, KindProbe},
+
+		// .env variants — an exact-name list can't keep up, the dotfile rule can.
+		{"env tilde", "/.env~", 404, browserUA, KindProbe},
+		{"env copy", "/.env_copy", 404, browserUA, KindProbe},
+		{"env backup2", "/.env.backup2", 404, browserUA, KindProbe},
+		{"nested env", "/backend/.env", 404, browserUA, KindProbe},
+
+		// A scanner sweeping the www host gets a 301, not a 404. Gating probes on
+		// 404 alone filed all of this as ordinary traffic.
+		{"probe redirected by www host", "/.env", 301, browserUA, KindProbe},
+		{"probe on a 500", "/wp-login.php", 500, browserUA, KindProbe},
+
+		// Bots.
+		{"googlebot", "/orgs", 200, "Googlebot/2.1 (+http://www.google.com/bot.html)", KindBot},
+		{"empty ua", "/orgs", 200, "", KindBot},
+		{"uptime monitor", "/", 200, "Better Uptime Monitor", KindBot},
+
+		// A scanner that also calls itself a bot is still a scanner.
+		{"bot ua on a probe path", "/wp-login.php", 404, "evilbot/1.0", KindProbe},
+
+		// Honest 404s stay visible as their own kind rather than being written off
+		// as attacks — a broken link is something worth fixing.
+		{"stale link", "/orgs/closed-club", 404, browserUA, KindNotFound},
+		{"typo", "/repeaterz", 404, browserUA, KindNotFound},
+		{"crawler on a dead link", "/orgs/gone", 404, "Googlebot/2.1", KindBot},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := classify(c.path, c.status, c.ua); got != c.want {
+				t.Errorf("classify(%q, %d, %q) = %q, want %q", c.path, c.status, c.ua, got, c.want)
+			}
+		})
+	}
+}
+
+// TestClassifyProductionCorpus runs the classifier over paths taken verbatim
+// from production traffic. The same corpus was checked against migration 0045's
+// SQL backfill, which found zero disagreements — keeping this list here is what
+// stops the Go rules and the SQL rules drifting apart unnoticed.
+func TestClassifyProductionCorpus(t *testing.T) {
+	probes := []string{
+		"/xmlrpc.php", "/info.php", "/test.php", "/.aws/credentials", "/actuator/env",
+		"/.vscode/sftp.json", "/api/gql", "/graphql/api", "/debug/default/view",
+		"/___proxy_subdomain_whm/login", "/stats/prometheus", "/.linode-cli",
+		"/config.js", "/aws.config.js", "/info", "/env", "/server", "/phpinfo",
+		"/.env.save", "/.env.bak", "/.env.prod", "/.env.dev", "/.env.old", "/.env.example",
+		"/app/.env", "/api/.env", "/backend/.env",
+		"/ecp/Current/exporttool/microsoft.exchange.ediscovery.exporttool.application",
+		// Backup/editor suffixes appended AFTER a real extension.
+		"/phpinfo.php~", "/phpinfo.php.save", "/config.json.save",
+		// Credential JSON under other stacks' names — no fixed list keeps up,
+		// which is why .json is scoped by prefix instead.
+		"/gcp-credentials.json", "/google-credentials.json", "/firebase-adminsdk.json",
+		"/aws-ses.json", "/aws.json", "/env.json", "/config/production.json",
+		"/appsettings.Development.json", "/appsettings.Production.json",
+		// Loose source and infra files.
+		"/aws-config.js", "/app.js", "/index.js", "/server.js", "/env.js", "/js/config.js",
+		"/settings.py", "/terraform.tfstate", "/Dockerfile", "/_environment", "/old/",
+		"/s/230313e28343e2333313e28363/_/;/META-INF/maven/com.atlassian.jira/jira-webapp-dist/pom.properties",
+	}
+	for _, p := range probes {
+		if got := classify(p, 404, browserUA); got != KindProbe {
+			t.Errorf("classify(%q, 404) = %q, want %q", p, got, KindProbe)
+		}
+	}
+
+	// Seen in production 404s and NOT attacks: these must stay visible as broken
+	// links so they can be fixed.
+	// Real people looking for pages we don't have, and crawlers on the auth host.
+	// These must stay readable as gaps to fill, not get written off as attacks.
+	notFound := []string{
+		"/about", "/contact", "/contact-us", "/login",
+		"/login/sitemap.xml", "/login/robots.txt",
+		"/orgs/example-mesh/config/edit", "/repeaters/Nf5YgD1sJw6k/console",
+	}
+	for _, p := range notFound {
+		if got := classify(p, 404, browserUA); got != KindNotFound {
+			t.Errorf("classify(%q, 404) = %q, want %q", p, got, KindNotFound)
+		}
+	}
+}
+
+// TestOwnRoutesAreNotProbes guards the signature list against our own URL space.
+// The rules match extensions, filenames, and dotfiles, and MeshTender genuinely
+// serves .json endpoints, has "config" all through the org routes, and has a
+// /console under each repeater — a loose rule here would file a member's stale
+// bookmark under "attacker".
+func TestOwnRoutesAreNotProbes(t *testing.T) {
+	ours := []string{
+		"/orgs/example-mesh/repeaters.json",
+		"/repeaters/Ab3xKp9QmR2t/config.json",
+		"/orgs/example-mesh/config",
+		"/orgs/example-mesh/config/profiles/new",
+		"/orgs/example-mesh/config/regions/12/area",
+		"/orgs/example-mesh/config/root-flood",
+		"/orgs/example-mesh/my-commands",
+		"/repeaters/Ab3xKp9QmR2t/console",
+		"/repeaters/Zq7WnT4vLh8c/console",
+		"/account/passkeys/rename",
+		"/api/login/discoverable/begin",
+		"/catalog/heltec-v3",
+		"/invite/:token",
+		"/build",
+	}
+	for _, p := range ours {
+		if isProbePath(p) {
+			t.Errorf("isProbePath(%q) = true, but that's one of our own routes", p)
+		}
+		if got := classify(p, 404, browserUA); got != KindNotFound {
+			t.Errorf("classify(%q, 404) = %q, want %q — a 404 on our own route is a broken link, not an attack", p, got, KindNotFound)
+		}
+	}
+}
+
+// TestConventionalPathsAreNotProbes: these 404 in production today and are worth
+// serving, not worth calling attacks. Filing them under "probe" would bury the
+// signal that we ought to add them — real iOS devices and search engines are
+// asking. .well-known in particular has to survive the dotfile rule.
+func TestConventionalPathsAreNotProbes(t *testing.T) {
+	conventional := []string{
+		"/apple-touch-icon.png",
+		"/apple-touch-icon-precomposed.png",
+		"/favicon.png",
+		"/sitemap.xml",
+		"/robots.txt",
+		"/llms.txt",
+		"/.well-known/security.txt",
+		"/.well-known/change-password",
+	}
+	for _, p := range conventional {
+		if isProbePath(p) {
+			t.Errorf("isProbePath(%q) = true, but that's a web convention worth serving", p)
+		}
+	}
+}

internal/core/analytics.go

@@ -3,6 +3,9 @@ package core
 import (
 	"net/http"
 	"time"
+
+	"github.com/jleight/meshtender/internal/analytics"
+	"github.com/jleight/meshtender/internal/store"
 )
 
 // analyticsBar is one day's column in the traffic charts; heights are percentages
@@ -33,31 +36,53 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) {
 		days = 90
 	}
 
-	daily, err := s.Store.AnalyticsDaily(r.Context(), days)
+	// Everything on the main dashboard reads the "visit" kind only. Scanners and
+	// crawlers are recorded too (see internal/analytics classify), but they'd
+	// dwarf the real numbers here — they get their own cards below.
+	daily, err := s.Store.AnalyticsDaily(r.Context(), days, analytics.KindVisit)
 	if err != nil {
 		s.ServerError(w, r, "could not load analytics", err)
 		return
 	}
-	surfaces, err := s.Store.AnalyticsBySurface(r.Context(), days)
+	surfaces, err := s.Store.AnalyticsBySurface(r.Context(), days, analytics.KindVisit)
 	if err != nil {
 		s.ServerError(w, r, "could not load analytics", err)
 		return
 	}
-	paths, err := s.Store.AnalyticsTopPaths(r.Context(), days, 20)
+	paths, err := s.Store.AnalyticsTopPaths(r.Context(), days, analytics.KindVisit, 20)
 	if err != nil {
 		s.ServerError(w, r, "could not load analytics", err)
 		return
 	}
-	hosts, err := s.Store.AnalyticsTopHosts(r.Context(), days, 15)
+	hosts, err := s.Store.AnalyticsTopHosts(r.Context(), days, analytics.KindVisit, 15)
 	if err != nil {
 		s.ServerError(w, r, "could not load analytics", err)
 		return
 	}
-	visitors, err := s.Store.AnalyticsTopVisitors(r.Context(), days, 15)
+	visitors, err := s.Store.AnalyticsTopVisitors(r.Context(), days, analytics.KindVisit, 15)
 	if err != nil {
 		s.ServerError(w, r, "could not load analytics", err)
 		return
 	}
+	kinds, err := s.Store.AnalyticsKindSummary(r.Context(), days)
+	if err != nil {
+		s.ServerError(w, r, "could not load analytics", err)
+		return
+	}
+	probePaths, err := s.Store.AnalyticsTopPaths(r.Context(), days, analytics.KindProbe, 10)
+	if err != nil {
+		s.ServerError(w, r, "could not load analytics", err)
+		return
+	}
+	botPaths, err := s.Store.AnalyticsTopPaths(r.Context(), days, analytics.KindBot, 10)
+	if err != nil {
+		s.ServerError(w, r, "could not load analytics", err)
+		return
+	}
+	byKind := make(map[string]store.KindStat, len(kinds))
+	for _, k := range kinds {
+		byKind[k.Kind] = k
+	}
 
 	var maxReq, maxVis, totalReq int64
 	for _, d := range daily {
@@ -101,16 +126,7 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) {
 		surfaceRows = append(surfaceRows, analyticsRow{Label: x.Surface, Value: x.Requests, W: barPct(x.Requests, maxSurface)})
 	}
 
-	var maxHits int64
-	for _, p := range paths {
-		if p.Hits > maxHits {
-			maxHits = p.Hits
-		}
-	}
-	pathRows := make([]analyticsRow, 0, len(paths))
-	for _, p := range paths {
-		pathRows = append(pathRows, analyticsRow{Label: p.Path, Value: p.Hits, W: barPct(p.Hits, maxHits)})
-	}
+	pathRows := toPathRows(paths)
 
 	var maxHostReq int64
 	for _, h := range hosts {
@@ -145,6 +161,7 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) {
 		})
 	}
 
+	probes, bots := byKind[analytics.KindProbe], byKind[analytics.KindBot]
 	s.Render(w, r, "analytics.html", map[string]any{
 		"Days":     days,
 		"Bars":     bars,
@@ -155,10 +172,36 @@ func (s *Handlers) pageAnalytics(w http.ResponseWriter, r *http.Request) {
 		"TotalReq": totalReq,
 		"TodayReq": todayReq,
 		"TodayVis": todayVis,
-		"HasData":  len(daily) > 0,
+		// Scanner and crawler traffic, held apart from the figures above so a
+		// wordlist replay can't read as an audience.
+		"ProbeReq":     probes.Requests,
+		"ProbeSources": probes.Sources,
+		"ProbePaths":   toPathRows(probePaths),
+		"BotReq":       bots.Requests,
+		"BotSources":   bots.Sources,
+		"BotPaths":     toPathRows(botPaths),
+		// 404s with no attack signature — broken links worth fixing. Filtered out
+		// of every figure above, so without this they'd be invisible entirely.
+		"NotFoundReq": byKind[analytics.KindNotFound].Requests,
+		"HasData":     len(daily) > 0 || len(kinds) > 0,
 	})
 }
 
+// toPathRows scales a set of path counts into labeled bars.
+func toPathRows(paths []store.PathStat) []analyticsRow {
+	var max int64
+	for _, p := range paths {
+		if p.Hits > max {
+			max = p.Hits
+		}
+	}
+	rows := make([]analyticsRow, 0, len(paths))
+	for _, p := range paths {
+		rows = append(rows, analyticsRow{Label: p.Path, Value: p.Hits, W: barPct(p.Hits, max)})
+	}
+	return rows
+}
+
 // visitorRow is one (daily-rotating) visitor hash for the "traffic by user" table.
 type visitorRow struct {
 	Visitor  string

internal/core/templates/analytics.html

@@ -43,6 +43,13 @@
   </div>
 </div>
 
+<div class="alert alert-info mt-3" role="alert">
+  <div>
+    Everything above counts <strong>real visits only</strong>. Vulnerability scanners, crawlers, and
+    404s are recorded too, but kept out of these figures — they're broken out below.
+  </div>
+</div>
+
 <!-- Charts -->
 <div class="row row-cards mt-1">
   <div class="col-lg-6">
@@ -122,6 +129,69 @@
     </div>
   </div>
 </div>
+
+<!-- Traffic held apart from the figures above -->
+<div class="row row-cards mt-1">
+  <div class="col-lg-6">
+    <div class="card" data-testid="probe-card">
+      <div class="card-header">
+        <div>
+          <h2 class="card-title">Probes &amp; scanners</h2>
+          <div class="card-subtitle">Requests for things we don't serve — other stacks' admin panels,
+            credential files, known exploit paths.</div>
+        </div>
+      </div>
+      <div class="card-body">
+        <div class="row mb-3">
+          <div class="col-6">
+            <div class="h1 mb-0" data-testid="probe-requests">{{.ProbeReq}}</div>
+            <div class="text-secondary">Requests · last {{.Days}}d</div>
+          </div>
+          <div class="col-6">
+            <div class="h1 mb-0">{{.ProbeSources}}</div>
+            <div class="text-secondary">Distinct sources</div>
+          </div>
+        </div>
+        {{if .ProbePaths}}{{range .ProbePaths}}{{template "value-bar" .}}{{end}}
+        {{else}}<div class="text-secondary">Nothing probed in this window.</div>{{end}}
+      </div>
+    </div>
+  </div>
+  <div class="col-lg-6">
+    <div class="card" data-testid="bot-card">
+      <div class="card-header">
+        <div>
+          <h2 class="card-title">Bots &amp; crawlers</h2>
+          <div class="card-subtitle">Clients that identify themselves as automated: search engines,
+            uptime monitors, link previewers.</div>
+        </div>
+      </div>
+      <div class="card-body">
+        <div class="row mb-3">
+          <div class="col-6">
+            <div class="h1 mb-0" data-testid="bot-requests">{{.BotReq}}</div>
+            <div class="text-secondary">Requests · last {{.Days}}d</div>
+          </div>
+          <div class="col-6">
+            <div class="h1 mb-0">{{.BotSources}}</div>
+            <div class="text-secondary">Distinct sources</div>
+          </div>
+        </div>
+        {{if .BotPaths}}{{range .BotPaths}}{{template "value-bar" .}}{{end}}
+        {{else}}<div class="text-secondary">No crawler traffic in this window. Bots are only recorded
+          from the point this was deployed.</div>{{end}}
+      </div>
+    </div>
+  </div>
+</div>
+
+<div class="card card-sm mt-1">
+  <div class="card-body">
+    <div class="h1 mb-0" data-testid="notfound-requests">{{.NotFoundReq}}</div>
+    <div class="text-secondary">404s with no attack signature · last {{.Days}}d — broken links worth fixing,
+      counted nowhere above.</div>
+  </div>
+</div>
 {{end}}
 <a class="back-link mt-3" href="/admin">{{template "icon-arrow-left" "me-1"}}Back to admin</a>
 {{end}}

internal/e2e/analytics_cards_test.go

@@ -0,0 +1,91 @@
+//go:build browser
+
+package e2e
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/chromedp/chromedp"
+
+	"github.com/jleight/meshtender/internal/store"
+)
+
+// TestAnalyticsKindCards drives the traffic dashboard in a real browser after
+// seeding one of every event kind. It asserts the split the whole `kind` column
+// exists for: a scanner replaying a wordlist is counted in its own card and is
+// absent from the visitor and busiest-pages figures, where it used to outweigh
+// every real reader. Also catches a CSP trip on the new markup, which no Go
+// handler test can see.
+func TestAnalyticsKindCards(t *testing.T) {
+	srv := newE2EServer(t)
+	// login()'s first account bootstraps the capabilities that reach /admin.
+	_, cookie := srv.login(t, "e2eadmin")
+
+	now := time.Now()
+	ev := func(path, kind, visitor string, status int) store.AnalyticsEvent {
+		return store.AnalyticsEvent{
+			Ts: now, Surface: "root", Host: "root.example", Path: path,
+			Method: "GET", Status: status, Kind: kind, Visitor: visitor,
+		}
+	}
+	evs := []store.AnalyticsEvent{
+		ev("/", "visit", "alice", 200),
+		ev("/docs", "visit", "bob", 200),
+		ev("/gone", "notfound", "alice", 404),
+		ev("/orgs", "bot", "googlebot", 200),
+		ev("/catalog", "bot", "googlebot", 200),
+	}
+	// One scanner, five paths — the shape that swamped the dashboard.
+	for _, p := range []string{"/.env", "/wp-login.php", "/.git/config", "/index.php", "/phpinfo.php"} {
+		evs = append(evs, ev(p, "probe", "scanner", 404))
+	}
+	if err := srv.store.InsertAnalyticsEvents(srv.ctx, evs); err != nil {
+		t.Fatalf("seed analytics events: %v", err)
+	}
+	// The path cards read the rollups, not raw events.
+	if err := srv.store.RollupAnalytics(srv.ctx); err != nil {
+		t.Fatalf("roll up analytics: %v", err)
+	}
+
+	ctx, cancel, watch := startBrowser(t)
+	defer cancel()
+
+	var probeReq, botReq, notFoundReq, totalReq, probeCard, pagesCard string
+	if err := chromedp.Run(ctx,
+		setSessionCookie(cookie),
+		chromedp.Navigate(srv.appURL+"/admin/analytics"),
+		chromedp.WaitVisible(`[data-testid="probe-card"]`, chromedp.ByQuery),
+		chromedp.Text(`[data-testid="probe-requests"]`, &probeReq, chromedp.ByQuery),
+		chromedp.Text(`[data-testid="bot-requests"]`, &botReq, chromedp.ByQuery),
+		chromedp.Text(`[data-testid="notfound-requests"]`, &notFoundReq, chromedp.ByQuery),
+		chromedp.Text(`[data-testid="probe-card"]`, &probeCard, chromedp.ByQuery),
+		// The summary tile and the busiest-pages card must both be visits-only.
+		chromedp.Text(`.card-sm .h1`, &totalReq, chromedp.ByQuery),
+		chromedp.Text(`.row-cards`, &pagesCard, chromedp.ByQuery),
+	); err != nil {
+		t.Fatalf("drive analytics page: %v", err)
+	}
+
+	for _, c := range []struct{ name, got, want string }{
+		{"probe requests", probeReq, "5"},
+		{"bot requests", botReq, "2"},
+		{"notfound requests", notFoundReq, "1"},
+		{"total requests (visits only)", totalReq, "2"},
+	} {
+		if strings.TrimSpace(c.got) != c.want {
+			t.Errorf("%s = %q, want %q", c.name, strings.TrimSpace(c.got), c.want)
+		}
+	}
+
+	// The scanner's paths belong in the probe card and nowhere else.
+	if !strings.Contains(probeCard, "/wp-login.php") {
+		t.Errorf("probe card doesn't list the probed paths:\n%s", probeCard)
+	}
+	if strings.Contains(pagesCard, "/wp-login.php") {
+		t.Error("a probed path leaked into the visit-facing cards — the kind filter isn't applied")
+	}
+
+	watch.assertClean(t)
+}

internal/marketing/templates/privacy.html

@@ -72,8 +72,10 @@
           the HTTP method, the response status, and a <em>visitor hash</em>. The hash is a salted
           SHA-256 of the day plus your IP address and user agent — it rotates every day, so the same
           visitor counts once per day and cannot be followed from one day to the next.
-          <strong>Your IP address is never written down</strong>, only mixed into that day's hash. Bot
-          traffic is skipped. This runs entirely on our own server; no analytics provider is involved.
+          <strong>Your IP address is never written down</strong>, only mixed into that day's hash.
+          Crawlers and vulnerability scanners are recorded the same way but counted separately, so
+          they don't inflate the visitor figures. This runs entirely on our own server; no analytics
+          provider is involved.
         </p>
 
         <h3>Diagnostics</h3>

internal/store/analytics.go

@@ -16,6 +16,7 @@ type AnalyticsEvent struct {
 	Path    string
 	Method  string
 	Status  int
+	Kind    string // "visit" | "probe" | "notfound" | "bot" — see internal/analytics
 	Visitor string
 }
 
@@ -26,11 +27,11 @@ func (s *Store) InsertAnalyticsEvents(ctx context.Context, evs []AnalyticsEvent)
 	}
 	rows := make([][]any, len(evs))
 	for i, e := range evs {
-		rows[i] = []any{e.Ts, e.Surface, e.Host, e.Path, e.Method, e.Status, e.Visitor}
+		rows[i] = []any{e.Ts, e.Surface, e.Host, e.Path, e.Method, e.Status, e.Kind, e.Visitor}
 	}
 	_, err := s.pool.CopyFrom(ctx,
 		pgx.Identifier{"analytics_events"},
-		[]string{"ts", "surface", "host", "path", "method", "status", "visitor"},
+		[]string{"ts", "surface", "host", "path", "method", "status", "kind", "visitor"},
 		pgx.CopyFromRows(rows))
 	if err != nil {
 		return fmt.Errorf("insert analytics events: %w", err)
@@ -45,20 +46,20 @@ func (s *Store) RollupAnalytics(ctx context.Context) error {
 	return s.inTx(ctx, func(tx pgx.Tx) error {
 		stmts := []string{
 			`DELETE FROM analytics_daily WHERE day >= (now() - interval '1 day')::date`,
-			`INSERT INTO analytics_daily (day, requests, visitors)
-			 SELECT ts::date, count(*), count(DISTINCT visitor)
+			`INSERT INTO analytics_daily (day, kind, requests, visitors)
+			 SELECT ts::date, kind, count(*), count(DISTINCT visitor)
 			 FROM analytics_events WHERE ts >= (now() - interval '1 day')::date
-			 GROUP BY ts::date`,
+			 GROUP BY ts::date, kind`,
 			`DELETE FROM analytics_daily_surface WHERE day >= (now() - interval '1 day')::date`,
-			`INSERT INTO analytics_daily_surface (day, surface, requests)
-			 SELECT ts::date, surface, count(*)
+			`INSERT INTO analytics_daily_surface (day, kind, surface, requests)
+			 SELECT ts::date, kind, surface, count(*)
 			 FROM analytics_events WHERE ts >= (now() - interval '1 day')::date
-			 GROUP BY ts::date, surface`,
+			 GROUP BY ts::date, kind, surface`,
 			`DELETE FROM analytics_daily_path WHERE day >= (now() - interval '1 day')::date`,
-			`INSERT INTO analytics_daily_path (day, path, hits)
-			 SELECT ts::date, path, count(*)
+			`INSERT INTO analytics_daily_path (day, kind, path, hits)
+			 SELECT ts::date, kind, path, count(*)
 			 FROM analytics_events WHERE ts >= (now() - interval '1 day')::date
-			 GROUP BY ts::date, path`,
+			 GROUP BY ts::date, kind, path`,
 		}
 		for _, q := range stmts {
 			if _, err := tx.Exec(ctx, q); err != nil {
@@ -116,13 +117,39 @@ type VisitorStat struct {
 	Requests int64
 }
 
-// AnalyticsTopHosts returns the busiest request hosts over the last `days` days,
-// read live from the raw events so the breakdown reflects current traffic.
-func (s *Store) AnalyticsTopHosts(ctx context.Context, days, limit int) ([]HostStat, error) {
+// KindStat is request volume and distinct sources for one event kind.
+type KindStat struct {
+	Kind     string
+	Requests int64
+	Sources  int64
+}
+
+// AnalyticsKindSummary returns per-kind totals over the last `days` days. It
+// reads raw events rather than the rollups so "sources" is a true distinct count
+// across the window — summing the daily rollup's visitor counts would instead
+// count a scanner that runs every day once per day.
+func (s *Store) AnalyticsKindSummary(ctx context.Context, days int) ([]KindStat, error) {
+	rows, err := s.pool.Query(ctx, `
+		SELECT kind, count(*), count(DISTINCT visitor) FROM analytics_events
+		WHERE ts >= now() - ($1::int * interval '1 day')
+		GROUP BY kind ORDER BY count(*) DESC`, days)
+	if err != nil {
+		return nil, fmt.Errorf("analytics kind summary: %w", err)
+	}
+	return collectRows(rows, func(r pgx.Row) (KindStat, error) {
+		var k KindStat
+		return k, r.Scan(&k.Kind, &k.Requests, &k.Sources)
+	})
+}
+
+// AnalyticsTopHosts returns the busiest request hosts of one kind over the last
+// `days` days, read live from the raw events so the breakdown reflects current
+// traffic.
+func (s *Store) AnalyticsTopHosts(ctx context.Context, days int, kind string, limit int) ([]HostStat, error) {
 	rows, err := s.pool.Query(ctx, `
 		SELECT host, count(*) FROM analytics_events
-		WHERE ts >= now() - ($1::int * interval '1 day')
-		GROUP BY host ORDER BY count(*) DESC LIMIT $2`, days, limit)
+		WHERE ts >= now() - ($1::int * interval '1 day') AND kind = $2
+		GROUP BY host ORDER BY count(*) DESC LIMIT $3`, days, kind, limit)
 	if err != nil {
 		return nil, fmt.Errorf("analytics top hosts: %w", err)
 	}
@@ -132,18 +159,20 @@ func (s *Store) AnalyticsTopHosts(ctx context.Context, days, limit int) ([]HostS
 	})
 }
 
-// AnalyticsTopVisitors returns the busiest visitor hashes over the last `days`
-// days, with the surface and most-recent path each was seen on. Hashes rotate
-// daily, so a heavy daily user appears once per day, not once overall.
-func (s *Store) AnalyticsTopVisitors(ctx context.Context, days, limit int) ([]VisitorStat, error) {
+// AnalyticsTopVisitors returns the busiest visitor hashes of one kind over the
+// last `days` days, with the surface and most-recent path each was seen on.
+// Hashes rotate daily, so a heavy daily user appears once per day, not once
+// overall. Filtering by kind is what keeps a scanner replaying a wordlist out of
+// the visitor list — it can outweigh every real reader combined.
+func (s *Store) AnalyticsTopVisitors(ctx context.Context, days int, kind string, limit int) ([]VisitorStat, error) {
 	rows, err := s.pool.Query(ctx, `
 		SELECT e.visitor, count(*) AS n,
 		       (array_agg(e.surface ORDER BY e.ts DESC))[1] AS surface,
 		       max(e.ts) AS last_seen,
 		       (array_agg(e.path ORDER BY e.ts DESC))[1] AS last_path
 		FROM analytics_events e
-		WHERE e.ts >= now() - ($1::int * interval '1 day')
-		GROUP BY e.visitor ORDER BY n DESC LIMIT $2`, days, limit)
+		WHERE e.ts >= now() - ($1::int * interval '1 day') AND e.kind = $2
+		GROUP BY e.visitor ORDER BY n DESC LIMIT $3`, days, kind, limit)
 	if err != nil {
 		return nil, fmt.Errorf("analytics top visitors: %w", err)
 	}
@@ -153,11 +182,12 @@ func (s *Store) AnalyticsTopVisitors(ctx context.Context, days, limit int) ([]Vi
 	})
 }
 
-// AnalyticsDaily returns the last `days` days of totals, oldest first.
-func (s *Store) AnalyticsDaily(ctx context.Context, days int) ([]DayStat, error) {
+// AnalyticsDaily returns the last `days` days of totals for one kind, oldest
+// first.
+func (s *Store) AnalyticsDaily(ctx context.Context, days int, kind string) ([]DayStat, error) {
 	rows, err := s.pool.Query(ctx, `
 		SELECT day, requests, visitors FROM analytics_daily
-		WHERE day >= (now()::date - $1::int) ORDER BY day`, days)
+		WHERE day >= (now()::date - $1::int) AND kind = $2 ORDER BY day`, days, kind)
 	if err != nil {
 		return nil, fmt.Errorf("analytics daily: %w", err)
 	}
@@ -167,11 +197,13 @@ func (s *Store) AnalyticsDaily(ctx context.Context, days int) ([]DayStat, error)
 	})
 }
 
-// AnalyticsBySurface returns request volume per surface over the last `days` days.
-func (s *Store) AnalyticsBySurface(ctx context.Context, days int) ([]SurfaceStat, error) {
+// AnalyticsBySurface returns request volume per surface for one kind over the
+// last `days` days.
+func (s *Store) AnalyticsBySurface(ctx context.Context, days int, kind string) ([]SurfaceStat, error) {
 	rows, err := s.pool.Query(ctx, `
 		SELECT surface, sum(requests) FROM analytics_daily_surface
-		WHERE day >= (now()::date - $1::int) GROUP BY surface ORDER BY sum(requests) DESC`, days)
+		WHERE day >= (now()::date - $1::int) AND kind = $2
+		GROUP BY surface ORDER BY sum(requests) DESC`, days, kind)
 	if err != nil {
 		return nil, fmt.Errorf("analytics by surface: %w", err)
 	}
@@ -181,11 +213,14 @@ func (s *Store) AnalyticsBySurface(ctx context.Context, days int) ([]SurfaceStat
 	})
 }
 
-// AnalyticsTopPaths returns the busiest paths over the last `days` days.
-func (s *Store) AnalyticsTopPaths(ctx context.Context, days, limit int) ([]PathStat, error) {
+// AnalyticsTopPaths returns the busiest paths of one kind over the last `days`
+// days — the pages people read for "visit", the wordlist being replayed for
+// "probe".
+func (s *Store) AnalyticsTopPaths(ctx context.Context, days int, kind string, limit int) ([]PathStat, error) {
 	rows, err := s.pool.Query(ctx, `
 		SELECT path, sum(hits) FROM analytics_daily_path
-		WHERE day >= (now()::date - $1::int) GROUP BY path ORDER BY sum(hits) DESC LIMIT $2`, days, limit)
+		WHERE day >= (now()::date - $1::int) AND kind = $2
+		GROUP BY path ORDER BY sum(hits) DESC LIMIT $3`, days, kind, limit)
 	if err != nil {
 		return nil, fmt.Errorf("analytics top paths: %w", err)
 	}

internal/store/analytics_test.go

@@ -11,9 +11,9 @@ func TestAnalyticsRollup(t *testing.T) {
 	now := time.Now()
 
 	evs := []AnalyticsEvent{
-		{Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Visitor: "v1"},
-		{Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Visitor: "v1"},
-		{Ts: now, Surface: "app", Host: "h", Path: "/b", Method: "GET", Status: 200, Visitor: "v2"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Kind: "visit", Visitor: "v1"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/a", Method: "GET", Status: 200, Kind: "visit", Visitor: "v1"},
+		{Ts: now, Surface: "app", Host: "h", Path: "/b", Method: "GET", Status: 200, Kind: "visit", Visitor: "v2"},
 	}
 	if err := st.InsertAnalyticsEvents(ctx, evs); err != nil {
 		t.Fatalf("insert: %v", err)
@@ -22,7 +22,7 @@ func TestAnalyticsRollup(t *testing.T) {
 		t.Fatalf("rollup: %v", err)
 	}
 
-	daily, err := st.AnalyticsDaily(ctx, 2)
+	daily, err := st.AnalyticsDaily(ctx, 2, "visit")
 	if err != nil {
 		t.Fatalf("daily: %v", err)
 	}
@@ -30,7 +30,7 @@ func TestAnalyticsRollup(t *testing.T) {
 		t.Fatalf("daily = %+v, want one day with 3 requests / 2 visitors", daily)
 	}
 
-	surf, err := st.AnalyticsBySurface(ctx, 2)
+	surf, err := st.AnalyticsBySurface(ctx, 2, "visit")
 	if err != nil {
 		t.Fatalf("surface: %v", err)
 	}
@@ -42,7 +42,7 @@ func TestAnalyticsRollup(t *testing.T) {
 		t.Fatalf("by surface = %v, want root=2 app=1", bySurface)
 	}
 
-	paths, err := st.AnalyticsTopPaths(ctx, 2, 10)
+	paths, err := st.AnalyticsTopPaths(ctx, 2, "visit", 10)
 	if err != nil {
 		t.Fatalf("paths: %v", err)
 	}
@@ -54,42 +54,180 @@ func TestAnalyticsRollup(t *testing.T) {
 	if err := st.RollupAnalytics(ctx); err != nil {
 		t.Fatalf("rollup again: %v", err)
 	}
-	daily, _ = st.AnalyticsDaily(ctx, 2)
+	daily, _ = st.AnalyticsDaily(ctx, 2, "visit")
 	if len(daily) != 1 || daily[0].Requests != 3 {
 		t.Fatalf("after re-rollup daily = %+v, want unchanged", daily)
 	}
 }
 
+// TestAnalyticsRollupSeparatesKinds is the regression this whole column exists
+// for: one scanner replaying a wordlist used to outweigh every real visitor in
+// the rollups. Its requests must still be stored and countable, just never mixed
+// into the visit figures.
+func TestAnalyticsRollupSeparatesKinds(t *testing.T) {
+	t.Parallel()
+	st, ctx := orgTestStore(t)
+	now := time.Now()
+
+	evs := []AnalyticsEvent{
+		{Ts: now, Surface: "root", Host: "h", Path: "/", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/docs", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/orgs", Method: "GET", Status: 200, Kind: "bot", Visitor: "googlebot"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/gone", Method: "GET", Status: 404, Kind: "notfound", Visitor: "alice"},
+	}
+	// One scanner, many paths, all in a day — the shape that swamped the charts.
+	for i, p := range []string{"/wp-login.php", "/.env", "/index.php", "/firebase-key.json", "/manager/html"} {
+		evs = append(evs, AnalyticsEvent{
+			Ts: now.Add(-time.Duration(i) * time.Second), Surface: "root", Host: "h",
+			Path: p, Method: "GET", Status: 404, Kind: "probe", Visitor: "scanner",
+		})
+	}
+	if err := st.InsertAnalyticsEvents(ctx, evs); err != nil {
+		t.Fatalf("insert: %v", err)
+	}
+	if err := st.RollupAnalytics(ctx); err != nil {
+		t.Fatalf("rollup: %v", err)
+	}
+
+	// Visits are unpolluted: 2 requests from 1 visitor, not 9 from 4.
+	visits, err := st.AnalyticsDaily(ctx, 2, "visit")
+	if err != nil {
+		t.Fatalf("daily visits: %v", err)
+	}
+	if len(visits) != 1 || visits[0].Requests != 2 || visits[0].Visitors != 1 {
+		t.Fatalf("visit day = %+v, want 2 requests / 1 visitor", visits)
+	}
+
+	// The scanner is still fully accounted for under its own kind.
+	probes, err := st.AnalyticsDaily(ctx, 2, "probe")
+	if err != nil {
+		t.Fatalf("daily probes: %v", err)
+	}
+	if len(probes) != 1 || probes[0].Requests != 5 || probes[0].Visitors != 1 {
+		t.Fatalf("probe day = %+v, want 5 requests / 1 source", probes)
+	}
+
+	bots, err := st.AnalyticsDaily(ctx, 2, "bot")
+	if err != nil {
+		t.Fatalf("daily bots: %v", err)
+	}
+	if len(bots) != 1 || bots[0].Requests != 1 {
+		t.Fatalf("bot day = %+v, want 1 request", bots)
+	}
+
+	// Per-kind path and visitor views read only their own kind.
+	probePaths, err := st.AnalyticsTopPaths(ctx, 2, "probe", 10)
+	if err != nil {
+		t.Fatalf("probe paths: %v", err)
+	}
+	if len(probePaths) != 5 {
+		t.Fatalf("probe paths = %+v, want the 5 probed paths", probePaths)
+	}
+	visitPaths, err := st.AnalyticsTopPaths(ctx, 2, "visit", 10)
+	if err != nil {
+		t.Fatalf("visit paths: %v", err)
+	}
+	for _, p := range visitPaths {
+		if p.Path == "/wp-login.php" {
+			t.Fatalf("a probed path leaked into the visit paths: %+v", visitPaths)
+		}
+	}
+
+	vis, err := st.AnalyticsTopVisitors(ctx, 2, "visit", 10)
+	if err != nil {
+		t.Fatalf("visitors: %v", err)
+	}
+	if len(vis) != 1 || vis[0].Visitor != "alice" {
+		t.Fatalf("top visitors = %+v, want only alice — the scanner must not rank here", vis)
+	}
+}
+
+// TestAnalyticsKindSummary: the cards need a true distinct-source count across
+// the window. Summing the daily rollups would count a scanner that runs every
+// day once per day, turning one attacker into a crowd.
+func TestAnalyticsKindSummary(t *testing.T) {
+	t.Parallel()
+	st, ctx := orgTestStore(t)
+	now := time.Now()
+
+	evs := []AnalyticsEvent{
+		{Ts: now, Surface: "root", Host: "h", Path: "/", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/docs", Method: "GET", Status: 200, Kind: "visit", Visitor: "bob"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/orgs", Method: "GET", Status: 200, Kind: "bot", Visitor: "googlebot"},
+		{Ts: now, Surface: "root", Host: "h", Path: "/gone", Method: "GET", Status: 404, Kind: "notfound", Visitor: "alice"},
+	}
+	// The same scanner across two days: 4 requests, but one source.
+	for i, p := range []string{"/.env", "/wp-login.php", "/.git/config", "/index.php"} {
+		evs = append(evs, AnalyticsEvent{
+			Ts: now.Add(-time.Duration(i) * 12 * time.Hour), Surface: "root", Host: "h",
+			Path: p, Method: "GET", Status: 404, Kind: "probe", Visitor: "scanner",
+		})
+	}
+	if err := st.InsertAnalyticsEvents(ctx, evs); err != nil {
+		t.Fatalf("insert: %v", err)
+	}
+
+	kinds, err := st.AnalyticsKindSummary(ctx, 7)
+	if err != nil {
+		t.Fatalf("kind summary: %v", err)
+	}
+	got := map[string]KindStat{}
+	for _, k := range kinds {
+		got[k.Kind] = k
+	}
+	if p := got["probe"]; p.Requests != 4 || p.Sources != 1 {
+		t.Errorf("probe = %+v, want 4 requests from 1 source", p)
+	}
+	if v := got["visit"]; v.Requests != 2 || v.Sources != 2 {
+		t.Errorf("visit = %+v, want 2 requests from 2 sources", v)
+	}
+	if b := got["bot"]; b.Requests != 1 || b.Sources != 1 {
+		t.Errorf("bot = %+v, want 1 request from 1 source", b)
+	}
+	if n := got["notfound"]; n.Requests != 1 {
+		t.Errorf("notfound = %+v, want 1 request", n)
+	}
+}
+
 func TestAnalyticsHostsAndVisitors(t *testing.T) {
 	t.Parallel()
 	st, ctx := orgTestStore(t)
 	now := time.Now()
 
 	evs := []AnalyticsEvent{
-		{Ts: now.Add(-time.Minute), Surface: "custom", Host: "1.2.3.4", Path: "/", Method: "GET", Status: 200, Visitor: "bot"},
-		{Ts: now, Surface: "custom", Host: "1.2.3.4", Path: "/robots.txt", Method: "GET", Status: 200, Visitor: "bot"},
-		{Ts: now, Surface: "app", Host: "app.x", Path: "/dash", Method: "GET", Status: 200, Visitor: "alice"},
+		{Ts: now.Add(-time.Minute), Surface: "custom", Host: "1.2.3.4", Path: "/", Method: "GET", Status: 200, Kind: "bot", Visitor: "bot"},
+		{Ts: now, Surface: "custom", Host: "1.2.3.4", Path: "/robots.txt", Method: "GET", Status: 200, Kind: "bot", Visitor: "bot"},
+		{Ts: now, Surface: "app", Host: "app.x", Path: "/dash", Method: "GET", Status: 200, Kind: "visit", Visitor: "alice"},
 	}
 	if err := st.InsertAnalyticsEvents(ctx, evs); err != nil {
 		t.Fatalf("insert: %v", err)
 	}
 
-	hosts, err := st.AnalyticsTopHosts(ctx, 2, 10)
+	hosts, err := st.AnalyticsTopHosts(ctx, 2, "bot", 10)
 	if err != nil {
 		t.Fatalf("hosts: %v", err)
 	}
-	if len(hosts) != 2 || hosts[0].Host != "1.2.3.4" || hosts[0].Requests != 2 {
-		t.Fatalf("top hosts = %+v, want 1.2.3.4=2 first", hosts)
+	if len(hosts) != 1 || hosts[0].Host != "1.2.3.4" || hosts[0].Requests != 2 {
+		t.Fatalf("top bot hosts = %+v, want 1.2.3.4=2", hosts)
 	}
 
-	vis, err := st.AnalyticsTopVisitors(ctx, 2, 10)
+	vis, err := st.AnalyticsTopVisitors(ctx, 2, "bot", 10)
 	if err != nil {
 		t.Fatalf("visitors: %v", err)
 	}
-	if len(vis) != 2 || vis[0].Visitor != "bot" || vis[0].Requests != 2 {
-		t.Fatalf("top visitors = %+v, want bot=2 first", vis)
+	if len(vis) != 1 || vis[0].Visitor != "bot" || vis[0].Requests != 2 {
+		t.Fatalf("top bot visitors = %+v, want bot=2", vis)
 	}
 	if vis[0].Surface != "custom" || vis[0].LastPath != "/robots.txt" {
 		t.Fatalf("bot visitor surface/last-path = %q/%q, want custom//robots.txt", vis[0].Surface, vis[0].LastPath)
 	}
+
+	// The human is reachable under her own kind, and only there.
+	human, err := st.AnalyticsTopVisitors(ctx, 2, "visit", 10)
+	if err != nil {
+		t.Fatalf("visit visitors: %v", err)
+	}
+	if len(human) != 1 || human[0].Visitor != "alice" {
+		t.Fatalf("visit visitors = %+v, want only alice", human)
+	}
 }

internal/store/migrations/0045_analytics_kind.sql

@@ -0,0 +1,95 @@
+-- +goose Up
+-- Classify each recorded request so scanner noise stops being counted as
+-- traffic. A single vulnerability scanner can post hundreds of 404s a day under
+-- a browser user agent, which previously landed it at the top of "top visitors"
+-- and buried the real ones. kind is decided at record time (internal/analytics
+-- classify.go): "visit" | "probe" | "notfound" | "bot".
+ALTER TABLE analytics_events ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit';
+CREATE INDEX analytics_events_kind_ts_idx ON analytics_events (kind, ts);
+
+-- The rollups gain the same dimension so each kind keeps its own daily history
+-- and the admin charts can read one kind without scanning raw events.
+ALTER TABLE analytics_daily DROP CONSTRAINT analytics_daily_pkey;
+ALTER TABLE analytics_daily ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit';
+ALTER TABLE analytics_daily ADD PRIMARY KEY (day, kind);
+
+ALTER TABLE analytics_daily_surface DROP CONSTRAINT analytics_daily_surface_pkey;
+ALTER TABLE analytics_daily_surface ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit';
+ALTER TABLE analytics_daily_surface ADD PRIMARY KEY (day, kind, surface);
+
+ALTER TABLE analytics_daily_path DROP CONSTRAINT analytics_daily_path_pkey;
+ALTER TABLE analytics_daily_path ADD COLUMN kind TEXT NOT NULL DEFAULT 'visit';
+ALTER TABLE analytics_daily_path ADD PRIMARY KEY (day, kind, path);
+
+-- Backfill history with the same rules the Go classifier applies, so the
+-- existing retention window isn't left misreporting scanners as visitors. Bots
+-- can't be backfilled — they were dropped before insert and never stored, and
+-- no user agent is retained to re-derive them.
+-- The gate is "not a 2xx", not "is a 404": a scanner sweeping the www host gets
+-- a 301 to the apex, and gating on 404 alone left all of that counted as real
+-- traffic. `.json` is absent from the extension list and generic names are
+-- matched as whole segments only — /orgs/{id}/repeaters.json,
+-- /repeaters/{id}/config.json and /repeaters/{id}/console are real routes.
+UPDATE analytics_events SET kind = CASE
+    WHEN status NOT BETWEEN 200 AND 299 AND (
+        -- an unfilled placeholder from the scanner's own template
+        path LIKE '%*%'
+        -- generic names that are only suspicious at the root of a host
+        OR lower(path) IN ('/api', '/info', '/env', '/server', '/phpinfo', '/console', '/console/',
+                           '/config.json', '/config.js', '/aws.config.js',
+                           '/server-status', '/server-info', '/v2/_catalog', '/old/')
+        -- file types we serve nowhere, in ANY segment: the Laravel Ignition RCE
+        -- arrives as /index.php/_ignition/... with the .php mid-path. Editor and
+        -- backup droppings may follow the real extension (/phpinfo.php.save,
+        -- /phpinfo.php~), so they're allowed to trail the match.
+        OR path ~* '\.(php|phps|php[357]|ini|env|ya?ml|sql|py|tfstate|properties|js|aspx?|axd|cgi|jspx?|action|bak|old|swp|save|orig|copy|dist|tmp)(~|\.(save|bak|old|orig|copy|dist|tmp|backup))*($|/)'
+        -- other stacks' credential and config files, as whole segments
+        OR path ~* '/(firebase-key\.json|credentials\.json|service-account\.json|secrets\.json|settings\.json|appsettings\.json|sftp\.json|package\.json|composer\.json|web\.config|id_rsa|id_dsa|dockerfile|backup\.(zip|tar\.gz))($|/)'
+        -- JSON outside the two prefixes where we actually serve it
+        OR (path ~* '\.json($|/)' AND path !~ '^/(orgs|repeaters)/')
+        -- scanner wordlist fragments: other stacks' panels and RCE entry points
+        OR path ~* '(wp-|wordpress|xmlrpc|phpmyadmin|/pma/|adminer|cgi-bin|/vendor/|autodiscover|/owa/|/ecp/|manager/html|/solr/|jenkins|actuator|telescope|eval-stdin|hnap1|graphql|/gql|_profiler|@vite|___proxy_subdomain|debug/default|_catalog|_ignition|webhook-waiting|stats/prometheus|/goform/|/boaform/|_environment|meta-inf)'
+        -- we serve no dotfiles; .well-known is the one real convention
+        OR (path ~ '/\.[^/]' AND path !~* '/\.well-known($|/)')
+    ) THEN 'probe'
+    WHEN status = 404 THEN 'notfound'
+    ELSE 'visit'
+END;
+
+-- Rebuild every rollup from the reclassified raw events. Raw retention (90d)
+-- covers the whole window the dashboard can display, so this is a complete
+-- rebuild rather than a partial correction.
+DELETE FROM analytics_daily;
+INSERT INTO analytics_daily (day, kind, requests, visitors)
+SELECT ts::date, kind, count(*), count(DISTINCT visitor) FROM analytics_events
+GROUP BY ts::date, kind;
+
+DELETE FROM analytics_daily_surface;
+INSERT INTO analytics_daily_surface (day, kind, surface, requests)
+SELECT ts::date, kind, surface, count(*) FROM analytics_events
+GROUP BY ts::date, kind, surface;
+
+DELETE FROM analytics_daily_path;
+INSERT INTO analytics_daily_path (day, kind, path, hits)
+SELECT ts::date, kind, path, count(*) FROM analytics_events
+GROUP BY ts::date, kind, path;
+
+-- +goose Down
+DELETE FROM analytics_daily WHERE kind <> 'visit';
+DELETE FROM analytics_daily_surface WHERE kind <> 'visit';
+DELETE FROM analytics_daily_path WHERE kind <> 'visit';
+
+ALTER TABLE analytics_daily_path DROP CONSTRAINT analytics_daily_path_pkey;
+ALTER TABLE analytics_daily_path DROP COLUMN kind;
+ALTER TABLE analytics_daily_path ADD PRIMARY KEY (day, path);
+
+ALTER TABLE analytics_daily_surface DROP CONSTRAINT analytics_daily_surface_pkey;
+ALTER TABLE analytics_daily_surface DROP COLUMN kind;
+ALTER TABLE analytics_daily_surface ADD PRIMARY KEY (day, surface);
+
+ALTER TABLE analytics_daily DROP CONSTRAINT analytics_daily_pkey;
+ALTER TABLE analytics_daily DROP COLUMN kind;
+ALTER TABLE analytics_daily ADD PRIMARY KEY (day);
+
+DROP INDEX analytics_events_kind_ts_idx;
+ALTER TABLE analytics_events DROP COLUMN kind;