MeshTender/internal/analytics/classify.go

package analytics

import "strings"

// Event kinds. Every recorded request gets exactly one, decided at record time
// by classify. They're stored on the row and on every rollup so the dashboard
// can read one kind at a time — scanner noise is kept, not discarded, but it
// never lands in the same bucket as a person visiting a page.
const (
	KindVisit    = "visit"    // a real request that hit a real route
	KindProbe    = "probe"    // a request for something only an attacker asks for
	KindNotFound = "notfound" // a 404 with no attack signature — likely a broken link
	KindBot      = "bot"      // a self-identified crawler or monitor
)

// probeExtensions are file types we serve nowhere. Matched against every path
// SEGMENT, not just the last one: the Laravel Ignition RCE arrives as
// /index.php/_ignition/execute-solution, where the ".php" sits mid-path.
var probeExtensions = []string{
	".php", ".phps", ".php3", ".php5", ".php7",
	".ini", ".env", ".yaml", ".yml", ".sql", ".py", ".tfstate", ".properties",
	".asp", ".aspx", ".axd", ".cgi", ".jsp", ".jspx", ".action",
	// .js is safe to claim: the only JavaScript we serve lives under /static/,
	// which skip() drops before anything is recorded, so a .js that reaches the
	// classifier is by definition not ours.
	".js",
	// Editor and backup droppings. These also arrive appended to a real
	// extension (/phpinfo.php.save, /config.json.save), which is why the check
	// below strips them and re-tests rather than only matching the tail.
	".bak", ".old", ".swp", ".save", ".orig", ".copy", ".dist", ".tmp",
}

// backupSuffixes get stripped from a segment before the extension check runs, so
// /phpinfo.php~ is recognized as the .php probe it is.
var backupSuffixes = []string{"~", ".save", ".bak", ".old", ".orig", ".copy", ".dist", ".tmp", ".backup"}

// probeNames match a path segment exactly — credential and config files that
// ship with other stacks. Matching the whole segment (never a substring) is what
// keeps our own /orgs/{id}/repeaters.json and /repeaters/{id}/config.json out of
// this bucket; note that bare "config.json" is deliberately absent for exactly
// that reason and is handled by rootProbes instead.
var probeNames = []string{
	"firebase-key.json", "credentials.json", "service-account.json",
	"secrets.json", "settings.json", "appsettings.json", "sftp.json",
	"package.json", "composer.json", "web.config",
	"id_rsa", "id_dsa", "backup.zip", "backup.tar.gz", "dockerfile",
}

// jsonRoots are the only path prefixes under which we serve JSON
// (/orgs/{id}/repeaters.json and /repeaters/{id}/config.json). A .json anywhere
// else is someone fishing for another stack's credentials — production alone
// turned up gcp-credentials.json, firebase-adminsdk.json, aws-ses.json and
// appsettings.Production.json, which no fixed list of names would have kept up
// with. Scoping by prefix rather than by name keeps a 404 on our own two
// endpoints readable as the broken link it is.
var jsonRoots = []string{"/orgs/", "/repeaters/"}

// rootProbes are generic names that are only suspicious at the root of a host.
// "/api" and "/console" are scanner bait; /api/login/begin and
// /repeaters/{id}/console are ours. Exact full-path matches only.
var rootProbes = []string{
	"/api", "/info", "/env", "/server", "/phpinfo", "/console", "/console/",
	"/config.json", "/config.js", "/aws.config.js",
	"/server-status", "/server-info", "/v2/_catalog", "/old/",
}

// probeSegments are fragments from the standard scanner wordlists — other
// stacks' admin panels, framework internals, and known RCE entry points. Each is
// specific enough not to collide with our own URL space.
var probeSegments = []string{
	"wp-", "wordpress", "xmlrpc", "phpmyadmin", "/pma/", "adminer", "cgi-bin",
	"/vendor/", "autodiscover", "/owa/", "/ecp/", "manager/html", "/solr/",
	"jenkins", "actuator", "telescope", "eval-stdin", "hnap1",
	"graphql", "/gql", "_profiler", "@vite", "___proxy_subdomain",
	"debug/default", "_catalog", "_ignition", "webhook-waiting",
	"stats/prometheus", "/goform/", "/boaform/", "_environment", "meta-inf",
}

// classify buckets one finished request.
//
// A probe is a request for something we don't serve, that only an attacker asks
// for. The status gate is "not a 2xx": a scanner sweeping the www host gets a
// 301 to the apex rather than a 404, and gating on 404 alone let all of that
// through as ordinary traffic. A 2xx means we really do serve the path, so the
// signature must be wrong and the request stays a visit — that direction is the
// safe one to be wrong in.
//
// The signatures only decide which flavor of non-2xx it was, so a miss can never
// hide a real request: it degrades to "notfound", still visible, just not
// attributed to an attacker. Bots are checked after probes because a scanner is
// free to put "bot" in its user agent, and what it asked for is better evidence
// than what it calls itself.
//
// The SQL backfill in migration 0045 mirrors these rules; it runs once over
// history and the two aren't kept in lockstep afterwards.
func classify(path string, status int, ua string) string {
	if !isSuccess(status) && isProbePath(path) {
		return KindProbe
	}
	if isBot(ua) {
		return KindBot
	}
	if status == 404 {
		return KindNotFound
	}
	return KindVisit
}

// isSuccess reports whether the response actually served the path.
func isSuccess(status int) bool { return status >= 200 && status < 300 }

// isProbePath reports whether a path looks like it came off a scanner wordlist.
// Case-insensitive: the same list gets replayed in every casing.
func isProbePath(path string) bool {
	p := strings.ToLower(path)

	// A literal "*" is an unfilled placeholder from the scanner's own template
	// (/workspaces/*, /webhook-waiting/*). No browser ever sends one.
	if strings.Contains(p, "*") {
		return true
	}
	for _, s := range rootProbes {
		if p == s {
			return true
		}
	}
	for _, s := range probeSegments {
		if strings.Contains(p, s) {
			return true
		}
	}
	jsonIsOurs := false
	for _, root := range jsonRoots {
		if strings.HasPrefix(p, root) {
			jsonIsOurs = true
			break
		}
	}

	for _, seg := range strings.Split(p, "/") {
		if seg == "" {
			continue
		}
		// We serve no dotfiles. .well-known is the one real convention, and
		// exempting it keeps security.txt and friends out of the attack bucket.
		if seg[0] == '.' && seg != ".well-known" {
			return true
		}
		if strings.HasSuffix(seg, ".json") && !jsonIsOurs {
			return true
		}
		for _, s := range probeNames {
			if seg == s {
				return true
			}
		}
		// Strip editor/backup droppings before testing the extension, so
		// /phpinfo.php.save and /phpinfo.php~ read as the .php probes they are.
		base := seg
		for changed := true; changed; {
			changed = false
			for _, s := range backupSuffixes {
				if trimmed, ok := strings.CutSuffix(base, s); ok && trimmed != "" {
					base, changed = trimmed, true
				}
			}
		}
		for _, s := range probeExtensions {
			if strings.HasSuffix(seg, s) || strings.HasSuffix(base, s) {
				return true
			}
		}
	}
	return false
}

// isBot reports whether the user agent identifies itself as automated. An empty
// user agent counts: every real browser sends one.
func isBot(ua string) bool {
	ua = strings.ToLower(ua)
	if ua == "" {
		return true
	}
	for _, s := range []string{"bot", "crawl", "spider", "slurp", "headless", "preview", "monitor"} {
		if strings.Contains(ua, s) {
			return true
		}
	}
	return false
}