diff --git a/docs/analytics.md b/docs/analytics.md index 17c3be5..1fed10d 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -24,7 +24,14 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with - **Initial page load**: `to` is the loaded path. This ping is what starts the visit and counts the entry page view — the document GET alone records - nothing, so bots and admin browsing never register. Reloads are not + nothing, so bots and admin browsing never register. JS-running crawlers + (Googlebot, GoogleOther, Applebot, ...) do ping, but their User-Agent + gives them away: pings whose UA matches `_is_bot_ua` (anything calling + itself a "bot", plus known exceptions such as GoogleOther) are ignored + server-side, and their document GETs land in the crawler list instead. + No source-IP verification is done: a spoofed bot UA merely lands in the + crawler stats, and scanners that probe telltale paths are caught by the + abuse rules regardless. Reloads are not visits: the ping is skipped (PerformanceNavigationTiming `reload`), so a refresh neither counts a second view nor logs a self-transition. The GET handler stashes a cross-origin https `Referer` (origin part only) and any @@ -74,8 +81,13 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with removing older versions after an update; without the flag only an existing file is used. - **Crawler hits**: every document GET is queued in RAM as a pending crawler - hit. If a ping from the same client arrives within 10 seconds the hit is - discarded; otherwise it is written to `crawlers`. Crawlers do not count as + hit — except idle-time link preloads from pagerite.js, which carry an + `x-pagerite-preload` header and are not tracked at all (the ping sent when + the user actually navigates to a preloaded page does the counting; forging + the header only hides a GET from the crawler stats, the path-based abuse + classification is unaffected). If a ping + from the same client arrives within 10 seconds the hit is discarded; + otherwise it is written to `crawlers`. Crawlers do not count as visits or views. The `Accept-Language` header is stored on the shared `Client` immediately; reverse-DNS host names and DB-IP geoip country/city are filled in asynchronously, just like for real visits. In diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js index 5d403bd..ae2587d 100644 --- a/frontend/src/pagerite.js +++ b/frontend/src/pagerite.js @@ -337,7 +337,10 @@ import "overlayscrollbars/overlayscrollbars.css"; } for (const url of urls) { if (pageCache.has(url)) continue; - fetch(url) + // x-pagerite-preload: idle cache warm-up, not a page view — the + // server excludes these GETs from analytics (the ping sent on actual + // navigation does the counting). + fetch(url, { headers: { "x-pagerite-preload": "1" } }) .then((r) => (r.ok && (r.headers.get("content-type") || "").includes("text/html") ? r.text() : "")) .then((html) => { if (html) pageCache.set(url, html); }) diff --git a/pagerite/analytics.py b/pagerite/analytics.py index b64fcf1..a58a204 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -5,7 +5,13 @@ ping on page load starts a visit, later pings extend it, and pings with no known session start a fresh one (missing data, not dropped). The document GET handler stashes the entry referer (external https origin) and any utm_* query parameters in in-memory IP tables, consumed when the ping -starts the visit; nothing is counted without a ping (bots stay invisible). +starts the visit; nothing is counted without a ping (plain bots that only +fetch documents end up in the crawler list). JS-running crawlers +(Googlebot, GoogleOther, Applebot, ...) do ping, but their UA gives them +away (``_is_bot_ua``) and their pings are ignored, so they land in the +crawler list too. Idle-time link preloads from pagerite.js carry an +``x-pagerite-preload`` header and are not tracked at all — the ping sent +when the user actually navigates does the counting. Admin clients ping with ``hide=1``, which records nothing and removes any visit the session accumulated before logging in. Scanner telltale 404s (dotpaths, *.php) classify the source IP as abuse; its hits — including @@ -239,6 +245,18 @@ def _utm_tags(query: str) -> dict[str, str]: _CRAWLER_TIMEOUT = timedelta(seconds=10) +#: UAs of JS-running crawlers, which would register as visitors on their +#: ping. Anything calling itself a "bot" matches; known crawlers without +#: that token (GoogleOther) are listed as extra alternates. No source +#: verification: a spoofed bot UA just lands in the crawler list, and +#: scanners that probe telltale paths are caught by the abuse rules anyway. +_BOT_UA = re.compile(r"bot|googleother", re.IGNORECASE) + + +def _is_bot_ua(ua: str) -> bool: + """True when the UA claims a crawler identity (Googlebot, Applebot, ...).""" + return bool(_BOT_UA.search(ua)) + #: Plain-404 count per IP that classifies it as abuse even without a #: telltale path hit. _ABUSE_404_THRESHOLD = 10 @@ -664,7 +682,10 @@ class Store: removed from the stats (the admin browsed anonymously before logging in). Nothing new is recorded. - Pings from IPs classified as abuse are ignored entirely. + Pings from IPs classified as abuse, and pings whose User-Agent + claims a JS-running crawler identity (``_is_bot_ua``), are ignored + entirely — the crawler's pending hits stay queued and flush to + ``data.crawlers`` normally. Returns the index of the new visit when one is created (or None) and the client hashes of any crawler hits flushed by this call, so callers @@ -685,6 +706,11 @@ class Store: return None, flushed if ip in self.data.abuse_ips: return None, flushed + if _is_bot_ua(ua): + # A JS-running crawler (Googlebot, GoogleOther, Applebot execute + # JS and ping): never a visit. Its pending crawler hits are + # kept and flush to ``data.crawlers`` normally. + return None, flushed # A real visitor ping cancels any pending crawler hits from this client. self.pending_crawlers = [ hit for hit in self.pending_crawlers if hit.client != client_hash diff --git a/pagerite/app.py b/pagerite/app.py index f5be174..95f22e8 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -857,8 +857,9 @@ def _track_entry(path: str, request: Request) -> list[bytes]: """Stash the referer/UTM tags and queue a pending crawler hit for the GET. Nothing is counted on the GET itself — the client's /_a ping starts the - visit, so bots never register as visits. (Admin clients ping too, but - with hide=1, which scrubs their session instead of recording it.) + visit, so bots never register as visits (JS-running crawlers ping too, + but the ping handler ignores known bot UAs). (Admin clients ping too, + but with hide=1, which scrubs their session instead of recording it.) The devserver's health probe (``GET /?from=devserver.py`` from ``127.0.0.1``) is ignored: it is not real traffic and would otherwise be @@ -868,6 +869,12 @@ def _track_entry(path: str, request: Request) -> list[bytes]: Returns the client hashes of any pending crawler hits flushed to persistent storage, so callers can schedule async geoip and reverse-DNS enrichment. """ + if request.headers.get("x-pagerite-preload"): + # Idle-time page-cache warm-up by pagerite.js, not a page view: the + # ping sent when the user actually navigates does the counting. + # (Forging the header only hides a GET from the crawler stats; the + # path-based abuse classification is unaffected.) + return [] if ( path == "" and str(request.url.query) == "from=devserver.py"