diff --git a/AGENTS.md b/AGENTS.md index 46b056c..d78adc9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `markdown.py` — markdown-it-py renderer. - `views.py` — shared page layout and rendering; theme/user-font resolution across `THEME_DIRS` / `FONT_DIRS` (cwd, site, platform data roots, then built-in `pagerite/themes/`, see `docs/themes-and-assets.md`). - `seed.py` — demo content, written only on first database creation. - - `analytics.py` — visit analytics collection (see `docs/analytics.md`). + - `analytics.py` — visit analytics collection (see `docs/analytics.md`). UA formatting/bot detection comes from the **uarite** package. - `frontend/src/` — Vue editor and public-page JS entries. - `main.js` — Vue editor app entry. - `analytics-main.js` — analytics page entry (mounts `AnalyticsView` at `/_a`). diff --git a/docs/analytics.md b/docs/analytics.md index f9d4d62..b130efe 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -63,8 +63,13 @@ Each `Client` record (shared by every event, keyed by hash): when a database is available, - `city` — city name from the DB-IP MMDB lookup, when available, - `ua` — raw `User-Agent` string, -- `ua_pretty` — compact display form of the UA (browser/OS/device) when - parsable, otherwise the raw string, +- `ua_pretty` — compact display form of the UA from `uarite.uaparse()`: + the crawler name for bots, with a category suffix only where a provider + runs crawlers of more than one kind (`GPTBot (AI)` vs + `OAI-SearchBot (search)`, `Googlebot (search)` vs `Google-Extended (AI)`; + single-kind providers stay plain: `Facebook`, `WhatsApp`), `Browser/major OS` on the desktop, the device where that is + the relevant information (iPhone reports its iOS version, Android + phones their model instead of the OS), otherwise the raw string, - `hide` — true for admin clients (`hide` message field): everything this client ever did is recorded but excluded from every statistic and from the viewer payload. This is the one flag set at record time — it is a client @@ -169,7 +174,9 @@ for misses. `_CRAWLER_TIMEOUT` (10 s) is a crawler hit — plain bots that only fetch documents never register as visits. JS-running crawlers (Googlebot, GoogleOther, Applebot, ...) do connect and send messages, but their UA - gives them away (`_is_bot_ua`): their messages are ignored at display + gives them away (`_is_bot_ua`, backed by `uarite.uaparse` — which + also knows the disguised ones: facebookexternalhit, Google-Extended, + WhatsApp, ...): their messages are ignored at display time, so their GETs never match and land in the crawler list too. Real- browser bots whose UA does not match are caught by engagement: a visit whose total reported reading time is under 5 seconds (`_MIN_VISIT_READ`; diff --git a/pagerite/analytics.py b/pagerite/analytics.py index a5199b0..78a1306 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -60,32 +60,17 @@ from urllib.parse import parse_qs, urlencode, urlparse import blake3 import msgspec -from ua_parser import parse +import uarite def _compact_user_agent(ua: str) -> str: """Format a User-Agent string into a compact display form. - Returns the original UA when the parser cannot identify the browser/OS. + See ``uarite.uaparse``: crawler name (with category) for bots, + ``Browser/major OS`` or the device for real browsers, the raw UA when + unrecognized. """ - if not ua or not ua.strip() or ua == "-": - return "" - r = parse(ua) - browser = r.user_agent.family if r.user_agent else None - ver = r.user_agent.major if r.user_agent else "" - os_name = r.os.family if r.os else None - dev = r.device.family if r.device else None - if browser in (None, "Other") and os_name in (None, "Other"): - return ua - if browser and browser != "Other": - browser = browser.split()[0] - else: - browser = "" - os_name = os_name if os_name and os_name != "Other" else "" - if dev in (None, "Other") or dev == browser: - dev = "" - parts = [f"{browser}/{ver}" if browser else "", os_name, dev] - return " ".join(p for p in parts if p).strip() + return uarite.uaparse(ua).pretty class Ping(msgspec.Struct, omit_defaults=True): @@ -418,17 +403,22 @@ _MIN_VISIT_READ = 5 _FAVICON_RETRY = timedelta(days=7) #: UAs of JS-running crawlers, which would register as visitors on their -#: activity messages. Anything calling itself a "bot" or "spider" matches; -#: known crawlers without those tokens (GoogleOther) are listed as extra -#: alternates. No source verification: a spoofed bot UA just lands in the -#: crawler list, and scanners that probe telltale paths are caught by the -#: abuse rules anyway. -_BOT_UA = re.compile(r"bot|spider|googleother", re.IGNORECASE) +#: activity messages. ``uarite`` knows the common crawlers +#: and link-preview fetchers (including disguised ones such as +#: facebookexternalhit and Google-Extended) plus any UA with a +#: bot/spider/crawler/scanner token. No source verification: a spoofed +#: bot UA just lands in the crawler list, and scanners that probe +#: telltale paths are caught by the abuse rules anyway. def _is_bot_ua(ua: str) -> bool: - """True when the UA claims a crawler identity (bot or spider).""" - return bool(_BOT_UA.search(ua)) + """True when the UA is not a regular browser. + + Every real browser registers as ``kind == "browser"``; anything else + (recognized crawler/previewer, generic bot token, or an unclassified + HTTP client such as httpx) is not a visitor. + """ + return uarite.uaparse(ua).kind != "browser" #: Plain-404 count per IP within ``_ABUSE_404_WINDOW`` that classifies it as diff --git a/pyproject.toml b/pyproject.toml index 215f3ae..0796c98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "platformdirs>=4.11.5", "pygments>=2.20.0", "python-slugify>=8.0.4", - "ua-parser>=1.0.2", + "uarite>=0.1.2", "zstandard>=0.25.0", ]