diff --git a/docs/analytics.md b/docs/analytics.md index b130efe..474649b 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -63,18 +63,27 @@ Each `Client` record (shared by every event, keyed by hash): when a database is available, - `city` — city name from the DB-IP MMDB lookup, when available, - `ua` — raw `User-Agent` string, -- `ua_pretty` — compact display form of the UA from `uarite.uaparse()`: - the crawler name for bots, with a category suffix only where a provider - runs crawlers of more than one kind (`GPTBot (AI)` vs - `OAI-SearchBot (search)`, `Googlebot (search)` vs `Google-Extended (AI)`; - single-kind providers stay plain: `Facebook`, `WhatsApp`), `Browser/major OS` on the desktop, the device where that is - the relevant information (iPhone reports its iOS version, Android - phones their model instead of the OS), otherwise the raw string, - `hide` — true for admin clients (`hide` message field): everything this client ever did is recorded but excluded from every statistic and from the viewer payload. This is the one flag set at record time — it is a client property, not a classification. +The viewer payload adds one display-time field to each client, never +persisted (stored records keep the default and old data always follows the +current uarite version): + +- `uarite` — the `uarite.UA` dataclass from parsing the raw UA + (`pretty`/`engine`/`os`/`provider`/`kind`/`url`): the crawler name for + bots, + with a category suffix only where a provider runs crawlers of more than + one kind (`GPTBot (AI)` vs `OAI-SearchBot (search)`, `Googlebot (search)` + vs `Google-Extended (AI)`; single-kind providers stay plain: `Facebook`, + `WhatsApp`), `Browser/major OS` on the desktop, the device where that is + the relevant information (iPhone reports its iOS version, Android phones + their model instead of the OS), otherwise the raw string; `url` is the + crawler's info page when uarite knows one (rendered as a 🔗 link after the + pretty UA in the viewer), `kind` drives the bot classification. + A reverse-DNS lookup is attempted for each new client and the result, when available, is stored as `host`; local/reserved/multicast addresses are skipped. If a DB-IP MMDB file (`dbip-*.mmdb` or `dbip-*.mmdb.gz`) is present diff --git a/pagerite/analytics.py b/pagerite/analytics.py index 78a1306..75bc1fe 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -60,17 +60,17 @@ from urllib.parse import parse_qs, urlencode, urlparse import blake3 import msgspec -import uarite +from uarite import UA, uaparse -def _compact_user_agent(ua: str) -> str: - """Format a User-Agent string into a compact display form. +def _display_client(client: Client) -> Client: + """Client copy with ``uarite`` filled in by the current uarite. - See ``uarite.uaparse``: crawler name (with category) for bots, - ``Browser/major OS`` or the device for real browsers, the raw UA when - unrecognized. + The parsed UA is a display-time field: stored records always carry the + default (None), so it never lands on disk, and old records always + follow current uarite rules. """ - return uarite.uaparse(ua).pretty + return msgspec.structs.replace(client, uarite=uaparse(client.ua)) class Ping(msgspec.Struct, omit_defaults=True): @@ -157,11 +157,14 @@ class Client(msgspec.Struct, omit_defaults=True): city: str = "" #: Raw User-Agent header. ua: str = "" - #: Compact display form of ``ua`` (browser/OS/device) when parsable. - ua_pretty: str = "" #: True for admin clients (hide=1 ping): everything this client ever did #: is excluded from all statistics and from the viewer payload. hide: bool = False + #: Display-time parsed UA (uarite.UA dataclass: pretty/engine/os/ + #: provider/kind/url). Set only on the display-payload copies by + #: ``_display_client`` — stored records keep the default, so it is never + #: persisted and old data always follows the current uarite version. + uarite: UA | None = None # --- Display DTOs ------------------------------------------------------- @@ -418,7 +421,7 @@ def _is_bot_ua(ua: str) -> bool: (recognized crawler/previewer, generic bot token, or an unclassified HTTP client such as httpx) is not a visitor. """ - return uarite.uaparse(ua).kind != "browser" + return uaparse(ua).kind != "browser" #: Plain-404 count per IP within ``_ABUSE_404_WINDOW`` that classifies it as @@ -550,7 +553,6 @@ class Store: self.data.clients[h] = Client( ip=ip, ua=ua, - ua_pretty=_compact_user_agent(ua), lang=lang, country=country, ) @@ -930,7 +932,7 @@ class Store: and not self._hidden(g.client) and ip_of.get(g.client, "") in abuse_ips ], - clients={h: c for h, c in data.clients.items() if not c.hide}, + clients={h: _display_client(c) for h, c in data.clients.items() if not c.hide}, favicons={ origin: f"/_f/{f.file}" for origin, f in data.favicons.items() diff --git a/pagerite/tracking.py b/pagerite/tracking.py index 568746d..e76579a 100644 --- a/pagerite/tracking.py +++ b/pagerite/tracking.py @@ -26,6 +26,7 @@ import httpx import msgspec from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect from fastapi.responses import Response +from uarite import uaparse from pagerite import analytics from pagerite.data import resolve @@ -443,7 +444,7 @@ async def activity_ws(ws: WebSocket) -> None: # already printed there): compact UA plus the browser's language tag. lang, _country = analytics._parse_accept_language(accept_language) ws.scope.setdefault("state", {})["log_extra"] = " ".join( - part for part in (analytics._compact_user_agent(ua), lang) if part + part for part in (uaparse(ua).pretty, lang) if part ) await ws.accept() try: