Raw access-log analytics storage with display-time classification

Replace the pre-classified store (visits/crawlers/abuse lists written at
collection time, plus abuse_ips and in-memory pending/session tables) with
a raw append-only log: one Get record per document GET (full path, true
HTTP status, referer origin, preload flag) and one Msg per /_ws activity
message.  Visitor/crawler/abuse classification, visit grouping (30-minute
inactivity gap), status/referer/UTM attribution and all aggregates are
derived in Store.display(), so future rule changes never invalidate stored
data.  The viewer payload keeps its exact shape.

Fixes structurally:

- Abuser 404s on slug-format paths showed up as "articles read": the
  not-found branch recorded the request twice through separate status
  plumbing.  Each request is now recorded once with its true status.
- 404 trail links never rendered red: cache-served navigations issue no
  GET and the only real GET (the idle preload) was discarded before status
  recording.  Preloads are now recorded with pre=True, never counted, and
  used for status attribution.
- formatAbuseRows merged a path's 404 probes and 200 reads into one entry;
  the collapse is now keyed by (path, status class).

Rule improvements enabled by the redesign:

- The plain-404 abuse threshold counts within a 1-hour sliding window, so
  long-time readers accumulating misses never classify (scanners spray).
- Hidden (admin) clients never trigger abuse classification: editing means
  visiting not-found pages.
- /.well-known/ probes (RFC 8615, e.g. Chrome devtools) are never abuse
  evidence; //foo-style empty path segments are an instant telltale.
- Visits can no longer open on an external exit URL; favicon fetches skip
  hidden clients' referers/exits.

Legacy analytics.json files are set aside as .bak-legacy on startup.

pagerite/pages.py

@@ -3,11 +3,11 @@
 ``GET /{path:path}`` resolves a slug path against the menu tree and renders
 the page (or a category placeholder, or 404); it must be registered AFTER
 the fastapi-vue asset routes so built frontend files win over content slugs
-(see app.py). Requests are recorded in analytics (crawler hits and 404s
-here, visits via the /_ws socket in tracking.py).
+(see app.py). Every served document is recorded raw in analytics (one
+access-log line with its true HTTP status; classification happens at
+display time — see pagerite/analytics.py).
 """
 
-import asyncio
 import logging
 from datetime import UTC, datetime
 from email.utils import format_datetime
@@ -22,16 +22,9 @@ from pagerite.state import (
     SITE_URL,
     _html_response,
     _is_reserved,
-    analytics_store,
     data,
 )
-from pagerite.tracking import (
-    _client_ip,
-    _enrich_client,
-    _query_suffix,
-    _schedule_client_enrichment,
-    _track_entry,
-)
+from pagerite.tracking import _record_get
 
 logger = logging.getLogger(__name__)
 
@@ -146,20 +139,13 @@ async def show_page(request: Request, path: str) -> Response:
     placeholder page (nav links point straight at its first child).
     """
     path = path.strip("/")
-    ua = request.headers.get("user-agent", "")
     accept_language = request.headers.get("accept-language", "")
     if path and _is_reserved(path):
         # Invalid slug shape: not a content URL, let FastAPI return its
         # built-in 404 instead of rendering an editable article page.
-        # Scanner telltales (dotpaths like /.env, *.php) classify the IP
-        # as abuse in analytics.
-        client_hash = analytics_store.track_404(
-            _client_ip(request),
-            ua,
-            f"/{path}{_query_suffix(request)}",
-            accept_language,
-        )
-        asyncio.create_task(_enrich_client(client_hash))
+        # Recorded like any other GET: telltale scanner paths (dotpaths
+        # like /.env, *.php) classify the IP as abuse at display time.
+        _record_get(request, status=404)
         raise HTTPException(404)
     chain = resolve(data.menu, path)
     node = chain[-1] if chain else None
@@ -189,8 +175,7 @@ async def show_page(request: Request, path: str) -> Response:
         if request.headers.get("if-none-match") == etag:
             return Response(status_code=304)
         if _is_trackable_path(path):
-            flushed = _track_entry(path, request)
-            _schedule_client_enrichment(flushed)
+            _record_get(request)
         return _html_response(
             request,
             "page",
@@ -220,8 +205,7 @@ async def show_page(request: Request, path: str) -> Response:
         )
         link_lang = i18n.base_tag(query_lang or "")
         if _is_trackable_path(path):
-            flushed = _track_entry(path, request, status=404)
-            _schedule_client_enrichment(flushed)
+            _record_get(request, status=404)
         return _html_response(
             request,
             "category",
@@ -241,13 +225,5 @@ async def show_page(request: Request, path: str) -> Response:
             if item.published:
                 return RedirectResponse(f"/{slug}")
     if _is_trackable_path(path):
-        client_hash = analytics_store.track_404(
-            _client_ip(request),
-            ua,
-            f"/{path}{_query_suffix(request)}",
-            accept_language,
-        )
-        asyncio.create_task(_enrich_client(client_hash))
-        flushed = _track_entry(path, request, status=404)
-        _schedule_client_enrichment(flushed)
+        _record_get(request, status=404)
     return _html_response(request, "not-found", path, 404)

pagerite/tracking.py

@@ -27,8 +27,9 @@ from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
 from fastapi.responses import Response
 
 from pagerite import analytics
+from pagerite.data import resolve
 from pagerite.files import _hash_name, file_store
-from pagerite.state import SITE_URL, _html_response, analytics_store
+from pagerite.state import SITE_URL, _html_response, analytics_store, data
 
 logger = logging.getLogger(__name__)
 
@@ -331,7 +332,7 @@ async def _broadcast_analytics() -> None:
     """Send the current analytics snapshot to every connected WS client."""
     if not _analytics_ws_clients:
         return
-    payload = analytics_store.display_json()
+    payload = analytics_store.display_json(_in_menu)
     closed = set()
     for ws in _analytics_ws_clients:
         try:
@@ -358,47 +359,51 @@ def _schedule_analytics_broadcast() -> None:
     )
 
 
-def _track_entry(path: str, request: Request, *, status: int = 200) -> list[bytes]:
-    """Stash the referer/UTM tags and queue a pending crawler hit for the GET.
+def _in_menu(path: str) -> bool:
+    """True when ``path`` ("/a/b" or "/") resolves to a real menu node.
 
-    Nothing is counted on the GET itself — the client's first /_ws message
-    starts the visit, so bots never register as visits (JS-running crawlers
-    connect too, but the WebSocket handler ignores known bot UAs).  (Admin
-    clients report too, but with hide, which flags their visit hidden: it is
-    recorded but excluded from all statistics and from the crawler list.)
+    Category placeholders return 404 but are real nodes: their GETs must not
+    count as misses in the display-time abuse classification.
+    """
+    return resolve(data.menu, path.strip("/")) is not None
+
+
+def _record_get(request: Request, *, status: int = 200) -> None:
+    """Record the document GET as one raw access-log line in analytics.
+
+    Nothing is classified here — the true HTTP status, the full request path
+    (query included), an external referer origin and the preload flag are
+    stored, and visitor/crawler/abuse classification happens at display time
+    (see analytics.Store.display).  Idle-time preloads from pagerite.js
+    (``x-pagerite-preload`` header) are recorded with ``pre=True``: never
+    counted, but a navigation later served from the in-memory page cache is
+    attributed this GET's status.
 
     The devserver's health probe (``GET /?from=devserver.py`` from
-    ``127.0.0.1``) is ignored: it is not real traffic and would otherwise be
-    logged as a crawler hit.  The root-path and localhost checks prevent
-    remote visitors from hiding traffic with the same query string.
-
-    Returns the client hashes of any pending crawler hits flushed to persistent
-    storage, so callers can schedule async geoip and reverse-DNS enrichment.
+    ``127.0.0.1``) is ignored: it is not real traffic.  The root-path and
+    localhost checks prevent remote visitors from forging the same query.
     """
-    if request.headers.get("x-pagerite-preload"):
-        # Idle-time page-cache warm-up by pagerite.js, not a page view: the
-        # activity message sent when the user actually navigates does the
-        # counting.
-        # (Forging the header only hides a GET from the crawler stats; the
-        # path-based abuse classification is unaffected.)
-        return []
     if (
-        path == ""
+        request.url.path == "/"
         and str(request.url.query) == "from=devserver.py"
         and _client_ip(request) == "127.0.0.1"
     ):
-        return []
+        return
     own_origin = SITE_URL or f"https://{urlparse(str(request.base_url)).netloc}"
-    full_path = f"{request.url.path}{_query_suffix(request)}"
-    return analytics_store.track_entry(
-        request.headers.get("referer", ""),
-        own_origin,
+    referer = request.headers.get("referer", "")
+    if analytics._origin(referer) in (None, own_origin):
+        referer = ""
+    client_hash = analytics_store.record_get(
         _client_ip(request),
         request.headers.get("user-agent", ""),
-        full_path,
-        request.headers.get("accept-language", ""),
+        f"{request.url.path}{_query_suffix(request)}",
         status=status,
+        referer=referer,
+        accept_language=request.headers.get("accept-language", ""),
+        pre=bool(request.headers.get("x-pagerite-preload")),
     )
+    if client_hash is not None:
+        _schedule_client_enrichment([client_hash])
 
 
 @router.get("/_a", response_model=None)
@@ -425,9 +430,10 @@ async def activity_ws(ws: WebSocket) -> None:
     Public, like the pages themselves (only /_api is gated); one connection
     follows a browsing session.  Messages are ``analytics.Ping`` structs as
     JSON text frames; ``to`` set is a navigation, ``read`` alone a
-    reading-time update.  The reverse-DNS and DB-IP geoip lookups happen in
-    background tasks so message handling is never delayed by slow DNS or
-    the first MMDB decompress.
+    reading-time update.  Everything is recorded raw — known bot UAs and
+    abusive IPs are filtered at display time, not here.  The reverse-DNS and
+    DB-IP geoip lookups happen in background tasks so message handling is
+    never delayed by slow DNS or the first MMDB decompress.
     """
     await ws.accept()
     ip = _client_ip(ws)
@@ -440,7 +446,7 @@ async def activity_ws(ws: WebSocket) -> None:
                 msg = msgspec.json.decode(text.encode(), type=analytics.Ping)
             except msgspec.DecodeError:
                 continue
-            visit_index, flushed_clients = analytics_store.ping(
+            new_client = analytics_store.record_msg(
                 msg.fr,
                 msg.to or None,
                 ip,
@@ -449,10 +455,8 @@ async def activity_ws(ws: WebSocket) -> None:
                 hide=msg.hide,
                 read=msg.read,
             )
-            if visit_index is not None:
-                visit = analytics_store.data.visits[visit_index]
-                asyncio.create_task(_enrich_client(visit.client))
-            _schedule_client_enrichment(flushed_clients)
+            if new_client is not None:
+                _schedule_client_enrichment([new_client])
             _schedule_favicon_fetch()
     except WebSocketDisconnect:
         pass
@@ -466,7 +470,7 @@ async def analytics_websocket(ws: WebSocket) -> None:
     endpoint. Powers the analytics viewer rendered at /_a.
     """
     await ws.accept()
-    await ws.send_text(analytics_store.display_json())
+    await ws.send_text(analytics_store.display_json(_in_menu))
     _analytics_ws_clients.add(ws)
     try:
         while True: