From 6d2ae104d71ef73bcea003d2fb22f19ba9f8d7c2 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Sat, 22 Aug 2026 18:23:38 +0000 Subject: [PATCH] Fix analytics classification: ignore bot-UA pings, skip preload GETs. JS-running crawlers (Googlebot, GoogleOther, Applebot) execute pagerite.js and send navigation pings, registering as visitors. Pings whose User-Agent matches _is_bot_ua (any "bot" token plus listed exceptions) are now ignored, so their document GETs flush to the crawler list as intended. No source verification: a spoofed bot UA merely lands in the crawler stats, and path-based abuse classification catches scanners regardless. Idle-time link preloads from pagerite.js were queued as pending crawler hits and flushed to the crawler list whenever the user navigated more than 10s later, so real visitors' subpage loads showed up as crawler hits. Preload fetches now carry an x-pagerite-preload header and the document GET handler skips tracking for them; the ping sent on actual navigation does the counting. --- docs/analytics.md | 18 +++++++++++++++--- frontend/src/pagerite.js | 5 ++++- pagerite/analytics.py | 30 ++++++++++++++++++++++++++++-- pagerite/app.py | 11 +++++++++-- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/docs/analytics.md b/docs/analytics.md index 17c3be5..1fed10d 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -24,7 +24,14 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with - **Initial page load**: `to` is the loaded path. This ping is what starts the visit and counts the entry page view — the document GET alone records - nothing, so bots and admin browsing never register. Reloads are not + nothing, so bots and admin browsing never register. JS-running crawlers + (Googlebot, GoogleOther, Applebot, ...) do ping, but their User-Agent + gives them away: pings whose UA matches `_is_bot_ua` (anything calling + itself a "bot", plus known exceptions such as GoogleOther) are ignored + server-side, and their document GETs land in the crawler list instead. + No source-IP verification is done: a spoofed bot UA merely lands in the + crawler stats, and scanners that probe telltale paths are caught by the + abuse rules regardless. Reloads are not visits: the ping is skipped (PerformanceNavigationTiming `reload`), so a refresh neither counts a second view nor logs a self-transition. The GET handler stashes a cross-origin https `Referer` (origin part only) and any @@ -74,8 +81,13 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with removing older versions after an update; without the flag only an existing file is used. - **Crawler hits**: every document GET is queued in RAM as a pending crawler - hit. If a ping from the same client arrives within 10 seconds the hit is - discarded; otherwise it is written to `crawlers`. Crawlers do not count as + hit — except idle-time link preloads from pagerite.js, which carry an + `x-pagerite-preload` header and are not tracked at all (the ping sent when + the user actually navigates to a preloaded page does the counting; forging + the header only hides a GET from the crawler stats, the path-based abuse + classification is unaffected). If a ping + from the same client arrives within 10 seconds the hit is discarded; + otherwise it is written to `crawlers`. Crawlers do not count as visits or views. The `Accept-Language` header is stored on the shared `Client` immediately; reverse-DNS host names and DB-IP geoip country/city are filled in asynchronously, just like for real visits. In diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js index 5d403bd..ae2587d 100644 --- a/frontend/src/pagerite.js +++ b/frontend/src/pagerite.js @@ -337,7 +337,10 @@ import "overlayscrollbars/overlayscrollbars.css"; } for (const url of urls) { if (pageCache.has(url)) continue; - fetch(url) + // x-pagerite-preload: idle cache warm-up, not a page view — the + // server excludes these GETs from analytics (the ping sent on actual + // navigation does the counting). + fetch(url, { headers: { "x-pagerite-preload": "1" } }) .then((r) => (r.ok && (r.headers.get("content-type") || "").includes("text/html") ? r.text() : "")) .then((html) => { if (html) pageCache.set(url, html); }) diff --git a/pagerite/analytics.py b/pagerite/analytics.py index b64fcf1..a58a204 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -5,7 +5,13 @@ ping on page load starts a visit, later pings extend it, and pings with no known session start a fresh one (missing data, not dropped). The document GET handler stashes the entry referer (external https origin) and any utm_* query parameters in in-memory IP tables, consumed when the ping -starts the visit; nothing is counted without a ping (bots stay invisible). +starts the visit; nothing is counted without a ping (plain bots that only +fetch documents end up in the crawler list). JS-running crawlers +(Googlebot, GoogleOther, Applebot, ...) do ping, but their UA gives them +away (``_is_bot_ua``) and their pings are ignored, so they land in the +crawler list too. Idle-time link preloads from pagerite.js carry an +``x-pagerite-preload`` header and are not tracked at all — the ping sent +when the user actually navigates does the counting. Admin clients ping with ``hide=1``, which records nothing and removes any visit the session accumulated before logging in. Scanner telltale 404s (dotpaths, *.php) classify the source IP as abuse; its hits — including @@ -239,6 +245,18 @@ def _utm_tags(query: str) -> dict[str, str]: _CRAWLER_TIMEOUT = timedelta(seconds=10) +#: UAs of JS-running crawlers, which would register as visitors on their +#: ping. Anything calling itself a "bot" matches; known crawlers without +#: that token (GoogleOther) are listed as extra alternates. No source +#: verification: a spoofed bot UA just lands in the crawler list, and +#: scanners that probe telltale paths are caught by the abuse rules anyway. +_BOT_UA = re.compile(r"bot|googleother", re.IGNORECASE) + + +def _is_bot_ua(ua: str) -> bool: + """True when the UA claims a crawler identity (Googlebot, Applebot, ...).""" + return bool(_BOT_UA.search(ua)) + #: Plain-404 count per IP that classifies it as abuse even without a #: telltale path hit. _ABUSE_404_THRESHOLD = 10 @@ -664,7 +682,10 @@ class Store: removed from the stats (the admin browsed anonymously before logging in). Nothing new is recorded. - Pings from IPs classified as abuse are ignored entirely. + Pings from IPs classified as abuse, and pings whose User-Agent + claims a JS-running crawler identity (``_is_bot_ua``), are ignored + entirely — the crawler's pending hits stay queued and flush to + ``data.crawlers`` normally. Returns the index of the new visit when one is created (or None) and the client hashes of any crawler hits flushed by this call, so callers @@ -685,6 +706,11 @@ class Store: return None, flushed if ip in self.data.abuse_ips: return None, flushed + if _is_bot_ua(ua): + # A JS-running crawler (Googlebot, GoogleOther, Applebot execute + # JS and ping): never a visit. Its pending crawler hits are + # kept and flush to ``data.crawlers`` normally. + return None, flushed # A real visitor ping cancels any pending crawler hits from this client. self.pending_crawlers = [ hit for hit in self.pending_crawlers if hit.client != client_hash diff --git a/pagerite/app.py b/pagerite/app.py index f5be174..95f22e8 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -857,8 +857,9 @@ def _track_entry(path: str, request: Request) -> list[bytes]: """Stash the referer/UTM tags and queue a pending crawler hit for the GET. Nothing is counted on the GET itself — the client's /_a ping starts the - visit, so bots never register as visits. (Admin clients ping too, but - with hide=1, which scrubs their session instead of recording it.) + visit, so bots never register as visits (JS-running crawlers ping too, + but the ping handler ignores known bot UAs). (Admin clients ping too, + but with hide=1, which scrubs their session instead of recording it.) The devserver's health probe (``GET /?from=devserver.py`` from ``127.0.0.1``) is ignored: it is not real traffic and would otherwise be @@ -868,6 +869,12 @@ def _track_entry(path: str, request: Request) -> list[bytes]: Returns the client hashes of any pending crawler hits flushed to persistent storage, so callers can schedule async geoip and reverse-DNS enrichment. """ + if request.headers.get("x-pagerite-preload"): + # Idle-time page-cache warm-up by pagerite.js, not a page view: the + # ping sent when the user actually navigates does the counting. + # (Forging the header only hides a GET from the crawler stats; the + # path-based abuse classification is unaffected.) + return [] if ( path == "" and str(request.url.query) == "from=devserver.py"