diff --git a/.gitignore b/.gitignore index 851b616..9a83443 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.lock *.kantadb pagerite.analytics.json +dbip-*.mmdb* /pagerite/frontend-build package-lock.json diff --git a/docs/analytics.md b/docs/analytics.md index a0f1a48..6118957 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -24,9 +24,9 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with nothing, so bots and admin browsing never register. Reloads are not visits: the ping is skipped (PerformanceNavigationTiming `reload`), so a refresh neither counts a second view nor logs a self-transition. The GET - handler only stashes a cross-origin https `Referer` (origin part only) in - an in-memory IP → referer table, consumed by the ping that starts the - visit; internal or absent referers never touch the table. + handler stashes a cross-origin https `Referer` (origin part only) and any + `utm_*` query parameters in in-memory IP tables, consumed by the ping that + starts the visit; internal or absent referers never touch the referer table. - **Internal fetch-navigations**: `to` is the target path, sent only after the swap actually happened (a failed swap falls back to a full load, whose initial ping counts the view instead — no gap, no double count). @@ -41,6 +41,19 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with - The server validates `to`: internal paths must be valid slug paths ("/" or `[a-z0-9_-]` segments), external ones are re-derived to the https origin and accepted only when the client sent exactly that. +- The initial ping also records the visitor's `User-Agent` and + `Accept-Language` headers. The first `Accept-Language` tag is stored as + `lang` (e.g. `en-us`) and its region subtag, if present, is stored as + an initial `country` (e.g. `US`). +- The visitor IP is stored. A reverse-DNS lookup is attempted for each new + visit and the result, when available, is cached in RAM and stored as + `host`; local/reserved/multicast addresses are skipped. +- If a DB-IP MMDB file (`dbip-*.mmdb` or `dbip-*.mmdb.gz`) is present in the + repository root, it is loaded at startup and used to look up a more accurate + `country`. The MMDB lookup and the reverse-DNS lookup run in background + tasks after the visit is stored, so the `/ _a` response is never delayed. + The decompressed `dbip-*.mmdb` file is kept in the repository root and + ignored by git. ## Visits and sessions @@ -49,17 +62,26 @@ There are no cookies. A visit is tied together by the (IP, User-Agent) pair direct peer): the first ping from a pair starts a new visit, subsequent pings extend it. Pings arriving with no known session (server restart) start a fresh visit from the first ping — treated as missing data rather -than dropped. The (IP, UA) → visit map and the IP → entry-referer table -are in-memory only; IPs are never persisted. +than dropped. The (IP, UA) → visit map and the IP → entry-referer/UTM +tables are in-memory only, but the IP and any resolvable reverse-DNS host +name are stored on the `Visit` record itself. Each `Visit` record: - `start` — timestamp of the first event, - `entry` — first page (path) seen, - `referer` — external https origin of the initial load, `""` for direct, +- `ip` — visitor IP address (first `X-Forwarded-For` hop, or direct peer), +- `host` — reverse-DNS host name for `ip` when resolvable, else `""`, - `trail` — everything seen afterwards in first-seen order: page paths and external exit origins. Re-visiting an already seen page (incl. the entry) does not append. +- `lang` — first `Accept-Language` tag, lowercased (e.g. `en-us`), +- `country` — two-letter country code. Initially derived from the + `Accept-Language` region subtag, but overwritten by the DB-IP MMDB result + when a database is available, +- `ua` — raw `User-Agent` string from the initial ping, +- `utm` — `utm_*` query parameters from the landing URL, as a dict. ## Aggregates diff --git a/frontend/src/AnalyticsView.vue b/frontend/src/AnalyticsView.vue index c9149a3..1099e66 100644 --- a/frontend/src/AnalyticsView.vue +++ b/frontend/src/AnalyticsView.vue @@ -8,7 +8,13 @@ // See docs/analytics.md for the data format. import { computed, onMounted, onUnmounted, ref, watch } from 'vue' import { RANGES } from './analytics/time.js' -import { calcTotalViews, formatRecentVisits } from './analytics/format.js' +import { + calcTotalViews, + countByField, + countUtmTags, + formatCounts, + formatVisitRows, +} from './analytics/format.js' import TransitionGraph from './TransitionGraph.vue' import VisitorCharts from './VisitorCharts.vue' @@ -55,7 +61,15 @@ watch(range, (r) => { } }) -const recentVisits = computed(() => formatRecentVisits(visits.value, pageTree.value)) +const visitRows = computed(() => formatVisitRows(visits.value, pageTree.value)) +const languageCounts = computed(() => countByField(visits.value, 'lang')) +const countryCounts = computed(() => countByField(visits.value, 'country')) +const utmTagCounts = computed(() => countUtmTags(visits.value)) +const hasBreakdowns = computed(() => + languageCounts.value.length > 0 + || countryCounts.value.length > 0 + || utmTagCounts.value.length > 0, +) @@ -177,36 +231,90 @@ const recentVisits = computed(() => formatRecentVisits(visits.value, pageTree.va } .totals strong { font-size: 1.5rem; } -.visits { - list-style: none; - margin: 0; - padding: 0; +.visit-table-wrap { + overflow-x: auto; } -.visits li { - display: flex; - gap: 1rem; - padding: 0.2rem 0; - border-bottom: 1px solid var(--line); -} -.visits .when { - flex-shrink: 0; - color: var(--muted); - font-variant-numeric: tabular-nums; -} -.visits .trail { + +.visit-table { + width: 100%; + border-collapse: collapse; font-family: monospace; - word-break: normal; + font-size: 0.82rem; + line-height: 1.3; +} + +.visit-table th, +.visit-table td { + padding: 0.25rem 0.5rem; + border-bottom: 1px solid var(--line); + text-align: left; + vertical-align: top; +} + +.visit-table th { + color: var(--muted); + font-weight: normal; + text-transform: lowercase; + position: sticky; + top: 0; + background: var(--bg, Canvas); +} + +.visit-table .when { + white-space: nowrap; + color: var(--muted); +} + +.visit-table .trail { + max-width: 20rem; overflow-wrap: break-word; } -.visits .trail a { + +.visit-table .trail a { color: var(--text); text-decoration: none; } -.visits .trail a:hover { color: var(--accent); } -.visits .trail a + a { + +.visit-table .trail a:hover { color: var(--accent); } + +.visit-table .trail a + a { margin-left: 0.5rem; } +.visit-table .ua { + max-width: 18rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.breakdowns { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr)); + gap: 1rem; +} + +.breakdown { + font-size: 0.9rem; +} + +.breakdown h3 { + margin: 0 0 0.3rem; + font-size: 0.9rem; + color: var(--muted); +} + +.breakdown p { + margin: 0; + line-height: 1.4; +} + +.breakdown .note { + font-size: 0.8rem; + color: var(--muted); + margin-top: 0.2rem; +} + .empty, .loading, .error { color: var(--muted); } .error { color: var(--error, #c00); } diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js index f1f6346..f322a92 100644 --- a/frontend/src/analytics/format.js +++ b/frontend/src/analytics/format.js @@ -52,3 +52,70 @@ export function formatRecentVisits(visits, pageTree, limit = 50) { .filter((v) => v.steps.length) .slice(0, limit) } + +/** + * Count distinct values of a visit field, sorted most-common first. + * Returns an array of [value, count] pairs. + */ +export function countByField(visits, field) { + const counts = {} + for (const v of visits || []) { + const value = v[field] + if (!value) continue + counts[value] = (counts[value] || 0) + 1 + } + return Object.entries(counts).sort((a, b) => b[1] - a[1]) +} + +/** + * Count UTM parameter occurrences across visits. Each distinct + * ``parameter: value`` pair is counted separately. Returns [pair, count]. + */ +export function countUtmTags(visits) { + const counts = {} + for (const v of visits || []) { + for (const [key, value] of Object.entries(v.utm || {})) { + const label = `${key}: ${value}` + counts[label] = (counts[label] || 0) + 1 + } + } + return Object.entries(counts).sort((a, b) => b[1] - a[1]) +} + +/** Format a list of [value, count] pairs for inline display. */ +export function formatCounts(entries) { + return entries.map(([value, count]) => `${value} (${count})`).join(', ') +} + +/** + * Format raw visit records as rows for a technical table. Returns objects + * with display strings; missing values become "—". ``trail`` joins page + * titles (when known) with " -> ". + */ +export function formatVisitRows(visits, pageTree) { + const titles = buildTitleMap(pageTree) + return [...(visits || [])].reverse().map((v) => { + const trail = [v.entry, ...(v.trail || [])] + .filter((p) => p?.startsWith('/')) + .map((p) => ({ + path: p, + slug: slugOf(p), + title: titles.get(p) || '', + })) + const utm = Object.entries(v.utm || {}) + .map(([k, value]) => `${k}=${value}`) + .join(', ') + const dash = (s) => (s || '—') + return { + when: new Date(v.start).toLocaleString(), + trail, + referer: dash(v.referer), + ip: dash(v.ip), + host: dash(v.host), + lang: dash(v.lang), + country: dash(v.country), + ua: dash(v.ua), + utm: utm || '—', + } + }) +} diff --git a/pagerite/analytics.py b/pagerite/analytics.py index 6b64cd7..d481df6 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -3,10 +3,12 @@ Events come from navigation pings POSTed to /_a by pagerite.js: the first ping on page load starts a visit, later pings extend it, and pings with no known session start a fresh one (missing data, not dropped). The document -GET handler only stashes the entry referer (external https origin) in an -in-memory IP -> referer table, consumed when the ping starts the visit; -nothing is counted without a ping (bots and admin browsing stay invisible). -The session map is in-memory only; IPs are never persisted. +GET handler stashes the entry referer (external https origin) and any +utm_* query parameters in in-memory IP tables, consumed when the ping +starts the visit; nothing is counted without a ping (bots and admin +browsing stay invisible). The session map is in-memory only. The visitor +IP and, when available, its reverse-DNS host name are stored on the visit +record itself. Data is a msgspec Struct JSON-dumped to its own file (not the kanta db), rewritten atomically on every recorded event. @@ -17,7 +19,7 @@ import re import tempfile from datetime import UTC, datetime from pathlib import Path -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse import msgspec @@ -34,7 +36,19 @@ class Visit(msgspec.Struct, omit_defaults=True): entry: str #: External https origin of the initial load, "" for direct visits. referer: str = "" + #: Visitor IP address (first X-Forwarded-For hop or direct peer). + ip: str = "" + #: Reverse-DNS host name for ``ip`` when resolvable, else "". + host: str = "" trail: list[str] = [] + #: First Accept-Language tag, lowercased (e.g. "en-us"). + lang: str = "" + #: Two-letter region subtag derived from ``lang`` (e.g. "US"), or "". + country: str = "" + #: Raw User-Agent header from the initial ping. + ua: str = "" + #: UTM query parameters from the landing URL, keyed by parameter name. + utm: dict[str, str] = {} class Analytics(msgspec.Struct, omit_defaults=True): @@ -80,6 +94,37 @@ def _internal_path(to: str) -> str | None: return None +def _parse_accept_language(value: str) -> tuple[str, str]: + """First Accept-Language tag and the region/country subtag if present. + + ``en-US, fr;q=0.9`` -> ("en-us", "US"). Wildcards and missing regions + produce an empty country. The region is intentionally approximate: + it reflects the browser's language preference, not geo-location. + """ + if not value: + return "", "" + tag = value.split(",")[0].split(";")[0].strip() + if not tag or tag == "*": + return "", "" + lang = tag.lower() + country = "" + # Region subtags follow the initial language tag (en-US, zh-Hans-CN). + # A bare two-letter tag such as "fr" is a language code, not a region. + for part in reversed(tag.split("-")[1:]): + if len(part) == 2 and part.isalpha(): + country = part.upper() + break + return lang, country + + +def _utm_tags(query: str) -> dict[str, str]: + """UTM parameters from a query string, keeping only the first value.""" + if not query: + return {} + parsed = parse_qs(query, keep_blank_values=True) + return {k: v[0] for k, v in parsed.items() if k.startswith("utm_")} + + class Store: """In-memory analytics data plus the (IP, UA) -> visit session map.""" @@ -97,6 +142,11 @@ class Store: #: one, stashed for the visit the client's initial ping starts. #: Internal or absent referers never touch the table. self.pending_referers: dict[str, str] = {} + #: ip -> utm_* query parameters from the latest document GET that + #: carried any, stashed for the visit the client's initial ping starts. + #: Only non-empty sets are stored, so a later parameter-less page + #: does not overwrite an earlier tagged landing URL. + self.pending_utms: dict[str, dict[str, str]] = {} def _save(self) -> None: """Rewrite the JSON file atomically (temp file + rename).""" @@ -113,9 +163,28 @@ class Store: def _count(self, table: dict[str, int], key: str) -> None: table[key] = table.get(key, 0) + 1 - def _new_visit(self, entry: str, referer: str, key: tuple[str, str]) -> Visit: + def _new_visit( + self, + entry: str, + referer: str, + key: tuple[str, str], + ip: str = "", + lang: str = "", + country: str = "", + ua: str = "", + utm: dict[str, str] | None = None, + ) -> Visit: now = datetime.now(UTC) - visit = Visit(start=now, entry=entry, referer=referer) + visit = Visit( + start=now, + entry=entry, + referer=referer, + ip=ip, + lang=lang, + country=country, + ua=ua, + utm=utm or {}, + ) self.data.visits.append(visit) self.sessions[key] = len(self.data.visits) - 1 self._count(self.data.site_visits, _bucket(now)) @@ -125,43 +194,90 @@ class Store: ) return visit - def entry_referer(self, referer: str, own_origin: str, ip: str) -> None: - """Stash the entry referer of a document GET for ping attribution. + def enrich_visit( + self, + index: int, + *, + host: str = "", + country: str = "", + ) -> None: + """Fill in host/geoip fields on an existing visit after async lookups.""" + if index < 0 or index >= len(self.data.visits): + return + visit = self.data.visits[index] + changed = False + if host and not visit.host: + visit.host = host + changed = True + if country: + visit.country = country + changed = True + if changed: + self._save() + + def entry_referer( + self, referer: str, own_origin: str, ip: str, query: str = "" + ) -> None: + """Stash the entry referer and UTM tags of a document GET for ping attribution. Nothing is counted here — the client's initial /_a ping starts the visit (only non-admin clients ping). Only a cross-origin https referer updates the table; an internal or absent referer leaves any - stashed origin untouched. + stashed origin untouched. UTM parameters are kept only when the + landing URL actually carries them, so a subsequent parameter-less page + does not erase an earlier tagged landing. """ - if not referer: - return - origin = _origin(referer) - if origin is None or origin == own_origin: - return - self.pending_referers[ip] = origin + if referer: + origin = _origin(referer) + if origin is not None and origin != own_origin: + self.pending_referers[ip] = origin + utms = _utm_tags(query) + if utms: + self.pending_utms[ip] = utms - def ping(self, from_: str, to: str, ip: str, ua: str) -> None: + def ping( + self, + from_: str, + to: str, + ip: str, + ua: str, + accept_language: str = "", + ) -> int | None: """Record a client navigation ping ({from, to} from pagerite.js). ``to`` is an internal path ("/...") or an https origin for exit links; anything else is ignored. The transition is always counted; the trail only grows on first sight of a page within the visit. A ping with no known session starts a fresh visit, consuming the - referer stashed by the document GET if there is one. + referer and UTM tags stashed by the document GET if there are any. + + Returns the index of the new visit when one is created, so callers + can enrich it later with non-blocking lookups (host, geoip country). """ if to.startswith("/") and not to.startswith("//"): target = _internal_path(to) or "" else: target = _origin(to) or "" if not target or (not to.startswith("/") and target != to): - return + return None key = (ip, ua) index = self.sessions.get(key) fr = (_internal_path(from_) or "(direct)") if from_ else "(direct)" if index is None or index >= len(self.data.visits): # No known session: the initial ping of a fresh page load (or # missing data after a server restart) — start a visit. - visit = self._new_visit(target, self.pending_referers.pop(ip, ""), key) + lang, country = _parse_accept_language(accept_language) + index = len(self.data.visits) + self._new_visit( + target, + self.pending_referers.pop(ip, ""), + key, + ip=ip, + lang=lang, + country=country, + ua=ua, + utm=self.pending_utms.pop(ip, {}), + ) else: visit = self.data.visits[index] now = datetime.now(UTC) @@ -172,3 +288,4 @@ class Store: if visit.entry != target and target not in visit.trail: visit.trail.append(target) self._save() + return index if index is not None and index < len(self.data.visits) else None diff --git a/pagerite/app.py b/pagerite/app.py index 2cbdb8d..01dfdac 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -12,13 +12,19 @@ walking the tree (``resolve``), moves are slot detach/attach (``find_slot``) with a fresh order key from the new siblings. """ +import asyncio +import gzip +import ipaddress import mimetypes import os import re +import shutil +import socket from collections.abc import AsyncIterator from contextlib import asynccontextmanager from datetime import UTC, datetime from email.utils import format_datetime +from functools import lru_cache from pathlib import Path from urllib.parse import urlparse @@ -51,6 +57,74 @@ ANALYTICS_PATH = Path( ) analytics_store = analytics.Store(ANALYTICS_PATH) + +# Repository root from this file's location (pagerite/app.py -> ..). +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _geoip_db_path() -> Path | None: + """Find a DB-IP MMDB in the repo root, preferring an already-decompressed + ``.mmdb`` over the matching ``.mmdb.gz``. Returns None if none is present. + """ + mmdb = sorted(_REPO_ROOT.glob("dbip-*.mmdb")) + if mmdb: + return mmdb[0] + gz = sorted(_REPO_ROOT.glob("dbip-*.mmdb.gz")) + if gz: + return gz[0] + return None + + +class GeoIP: + """Lazy DB-IP MMDB reader. Call ``_load()`` once at startup before + concurrent requests arrive; ``country()`` is read-only and safe to call + from ``asyncio.to_thread`` workers afterwards. + """ + + def __init__(self) -> None: + self._reader: object | None = None + + def _decompress(self, source: Path, target: Path) -> None: + if target.exists(): + return + tmp = target.with_suffix(target.suffix + ".tmp") + with gzip.open(source, "rb") as src, open(tmp, "wb") as dst: + shutil.copyfileobj(src, dst) + os.replace(tmp, target) + + def _load(self) -> None: + if self._reader is not None: + return + source = _geoip_db_path() + if source is None: + return + if source.suffix == ".gz": + target = source.with_suffix("") + self._decompress(source, target) + source = target + try: + import maxminddb + + self._reader = maxminddb.open_database(str(source)) + except Exception: + pass + + def country(self, ip: str) -> str: + """Two-letter ISO country code for ``ip``, or "" when unavailable.""" + if not ip or self._reader is None: + return "" + try: + rec = self._reader.get(ip) + if rec: + return (rec.get("country") or {}).get("iso_code", "") + except Exception: + pass + return "" + + +_geoip = GeoIP() + + # Our own data root; kanta edits it in place, reads are plain attribute access. data = Data() kanta = Kanta(DB_PATH, data) @@ -150,10 +224,13 @@ def _seed(data: Data) -> None: @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: - """Open the database, migrate legacy content, load assets.""" + """Open the database, migrate legacy content, load assets, load GeoIP.""" await kanta.open() _migrate_legacy() await frontend.load() + # Decompress/open the DB-IP MMDB once at startup. Lookups are then + # read-only and safe to run in background ``to_thread`` workers. + await asyncio.to_thread(_geoip._load) yield await kanta.close() @@ -510,6 +587,43 @@ def _client_ip(request: Request) -> str: return forwarded or (request.client.host if request.client else "") +@lru_cache(maxsize=4096) +def _cached_ptr(ip: str) -> str: + """Reverse-DNS lookup with in-RAM LRU cache. Returns the host name or "".""" + if not ip: + return "" + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return "" + if addr.is_private or addr.is_loopback or addr.is_reserved or addr.is_multicast or addr.is_link_local: + return "" + try: + host, _, _ = socket.gethostbyaddr(ip) + except socket.herror: + return "" + return host + + +async def _lookup_host(ip: str) -> str: + """Async wrapper around ``_cached_ptr``; runs the blocking lookup in a thread.""" + return await asyncio.to_thread(_cached_ptr, ip) + + +async def _geoip_country(ip: str) -> str: + """Async wrapper around the DB-IP MMDB lookup.""" + return await asyncio.to_thread(_geoip.country, ip) + + +async def _enrich_visit(index: int, ip: str) -> None: + """Run non-blocking reverse-DNS and geoip enrichment for a new visit.""" + if not ip: + return + host = await _lookup_host(ip) + country = await _geoip_country(ip) + analytics_store.enrich_visit(index, host=host, country=country) + + class AnalyticsPing(BaseModel): """Navigation ping from pagerite.js (see docs/analytics.md).""" @@ -519,22 +633,35 @@ class AnalyticsPing(BaseModel): @app.post("/_a", status_code=204) async def analytics_ping(ping: AnalyticsPing, request: Request) -> None: - """Record a navigation ping ({fr, to}); fire-and-forget, never fails.""" - analytics_store.ping( - ping.fr, ping.to, _client_ip(request), + """Record a navigation ping ({fr, to}); fire-and-forget, never fails. + + The reverse-DNS and DB-IP geoip lookups happen in a background task so + the response is never delayed by slow DNS or the first MMDB decompress. + """ + ip = _client_ip(request) + index = analytics_store.ping( + ping.fr, + ping.to, + ip, request.headers.get("user-agent", ""), + request.headers.get("accept-language", ""), ) + if index is not None: + asyncio.create_task(_enrich_visit(index, ip)) def _track_entry(path: str, request: Request) -> None: - """Stash the referer of the document GET for the initial ping. + """Stash the referer and UTM tags of the document GET for the initial ping. Nothing is counted on the GET itself — the client's /_a ping starts the visit, so bots and admin browsing never register. """ own_origin = f"https://{urlparse(str(request.base_url)).netloc}" analytics_store.entry_referer( - request.headers.get("referer", ""), own_origin, _client_ip(request) + request.headers.get("referer", ""), + own_origin, + _client_ip(request), + str(request.url.query), ) diff --git a/pyproject.toml b/pyproject.toml index f90e536..c69e12d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "html5tagger>=2.0.0", "kanta>=0.8.1", "markdown-it-py>=4.2.0", + "maxminddb>=3.1.1", "mdit-py-plugins>=0.6.1", "pygments>=2.20.0", ]