From 97bde492961f2ad4d7b1ecd59b676d0c9f7785d3 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 23 Sep 2026 07:12:15 +0000 Subject: [PATCH] Record robots.txt/sitemap.xml GETs, emoji-mark machinery steps in trails The robots and sitemap routes never called _record_get, so those accesses were invisible. Recorded like page GETs, they surface as crawler hits at display time (no activity message ever follows). In the trail rendering, known non-article machinery paths (robots, sitemap, future feed paths) get an emoji marker instead of a bare slug. --- docs/analytics.md | 9 +++++++-- frontend/src/analytics/format.js | 21 +++++++++++++++++++++ pagerite/analytics.py | 5 ++++- pagerite/pages.py | 8 +++++++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/docs/analytics.md b/docs/analytics.md index 64704b0..f2a7078 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -9,7 +9,9 @@ from the kanta content database, path from `PAGERITE_ANALYTICS` (default: `Favicon`), the `Store` (raw log + atomic JSON persistence) and `Store.display()`, where **all** classification happens. - `pagerite/pages.py` β€” records every served document as one raw GET line - (`_record_get`, in `pagerite/tracking.py`) with its true HTTP status. + (`_record_get`, in `pagerite/tracking.py`) with its true HTTP status, plus + the `/robots.txt` and `/sitemap.xml` machinery GETs (never followed by an + activity message, they surface as crawler hits). - `pagerite/tracking.py` β€” the `/_ws` activity WebSocket, and `WebSocket /_api/ws/analytics` (admin-gated like every `/_api` endpoint). - `frontend/src/pagerite.js` β€” the client activity channel and the πŸ“Š pen. @@ -211,7 +213,10 @@ for misses. are caught by the abuse rules regardless. In the viewer, crawler hits are grouped by client hash and shown as a trail of pages, preceded by the referer when there is one (rendered with its favicon like visit - referers). The crawler table lists the most recent crawler first, with + referers). Non-article machinery GETs (`/robots.txt`, `/sitemap.xml` β€” + and feed paths such as `/rss.xml` once those routes exist) appear as + emoji-marked steps (πŸ€– / πŸ—ΊοΈ / πŸ“‘) so they stand out from article steps. + The crawler table lists the most recent crawler first, with the most active as a tie-breaker. - **Abuse (scanner) hits**: a 404 on a telltale path β€” an empty URL segment (`//foo` β€” no real client generates those), any segment starting with a diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js index b762332..aa26065 100644 --- a/frontend/src/analytics/format.js +++ b/frontend/src/analytics/format.js @@ -160,9 +160,30 @@ function externalOrigin(url) { } } +// Non-article machinery paths shown in trails with an emoji marker: +// recorded like page GETs but fetched by crawlers/feed readers, so they +// surface in the crawler rows. Feed paths are pre-registered for the +// future RSS/Atom routes. +const MACHINE_STEPS = { + '/robots.txt': ['πŸ€–', 'robots.txt'], + '/sitemap.xml': ['πŸ—ΊοΈ', 'sitemap.xml'], + '/rss.xml': ['πŸ“‘', 'rss.xml'], + '/atom.xml': ['πŸ“‘', 'atom.xml'], + '/feed': ['πŸ“‘', 'feed'], + '/feed.xml': ['πŸ“‘', 'feed.xml'], +} + /** Format one trail step: an internal page or an external https origin. */ function stepOf(path, titles) { if (path?.startsWith('/')) { + // Known non-article machinery GETs (fetched by crawlers and feed + // readers, recorded like page GETs): emoji-marked so they stand out + // from article steps in the trails. + const machine = MACHINE_STEPS[path] + if (machine) { + const [emoji, name] = machine + return { path, slug: `${emoji} ${name}`, title: name, external: false, machine: true } + } return { path, slug: slugOf(path), title: titles.get(path) || '', external: false, home: path === '/' } } if (path?.startsWith('https://')) { diff --git a/pagerite/analytics.py b/pagerite/analytics.py index 0ed50c0..45ad4f5 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -5,7 +5,10 @@ to ``Analytics.gets`` as a raw access-log line (path with query string, true HTTP status, external referer origin, preload flag, rendered content language) and every pagerite.js activity message from the /_ws WebSocket is appended to ``Analytics.msgs`` -(navigations ``fr`` -> ``to`` and active reading-time updates). Nothing is +(navigations ``fr`` -> ``to`` and active reading-time updates). The +non-document machinery GETs ``/robots.txt`` and ``/sitemap.xml`` are +recorded the same way: never followed by an activity message, they surface +as crawler hits at display time. Nothing is classified when it is recorded: whether a client turns out to be a reader, a crawler or a scanner is decided by ``Store.display()`` from the raw events, so the stored data survives any future change to the classification diff --git a/pagerite/pages.py b/pagerite/pages.py index dcd3898..b677f06 100644 --- a/pagerite/pages.py +++ b/pagerite/pages.py @@ -5,7 +5,9 @@ the page (or a category placeholder, or 404); it must be registered AFTER the fastapi-vue asset routes so built frontend files win over content slugs (see app.py). Every served document is recorded raw in analytics (one access-log line with its true HTTP status; classification happens at -display time β€” see pagerite/analytics.py). +display time β€” see pagerite/analytics.py), as are robots.txt and sitemap.xml +fetches (they surface as crawler hits, since no activity message ever +follows them). """ import logging @@ -110,6 +112,9 @@ async def sitemap(request: Request) -> Response: ) lines.append("") + # Recorded like a page GET: never followed by an activity message, so + # it lands in the crawler list at display time (docs/analytics.md). + _record_get(request) return Response( "\n".join(lines), media_type="application/xml", @@ -124,6 +129,7 @@ async def robots_txt(request: Request) -> Response: the sitemap.""" base = SITE_URL or str(request.base_url).rstrip("/") body = f"User-agent: *\nAllow: /\nDisallow: /auth/\nDisallow: /_api\nSitemap: {base}/sitemap.xml\n" + _record_get(request) return Response( body, media_type="text/plain",