LeoVasanko/pagerite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Collect analytics over a /_ws WebSocket instead of POST /_a pings

Browse Source
This commit is contained in:
LeoVasanko
2026-08-29 21:20:04 +00:00
parent 447a565b05
commit 31895065f8
4 changed files with 255 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pagerite/analytics.py
+53 -26
View File
@@ -1,18 +1,23 @@
"""Server-side visit analytics (collection only; see docs/analytics.md).
Events come from navigation pings POSTed to /_a by pagerite.js: the first
ping on page load starts a visit, later pings extend it, and pings with no
known session start a fresh one (missing data, not dropped). The document
Events come from pagerite.js over the /_ws WebSocket (``Ping`` messages as
JSON text frames): the first navigation message on page load starts a
visit, later messages extend it, and messages with no known session start a
fresh one (missing data, not dropped). Active reading time is reported as
frequent ``read`` updates while the user is active; the times are
cumulative per trail item, so a disconnect simply leaves the last logged
time in place. The document
GET handler stashes the entry referer (external https origin) and any
utm_* query parameters in in-memory IP tables, consumed when the ping
starts the visit; nothing is counted without a ping (plain bots that only
fetch documents end up in the crawler list). JS-running crawlers
(Googlebot, GoogleOther, Applebot, ...) do ping, but their UA gives them
away (``_is_bot_ua``) and their pings are ignored, so they land in the
utm_* query parameters in in-memory IP tables, consumed when the first
message starts the visit; nothing is counted without a message (plain bots
that only fetch documents end up in the crawler list). JS-running crawlers
(Googlebot, GoogleOther, Applebot, ...) do connect and send messages, but
their UA gives them
away (``_is_bot_ua``) and their messages are ignored, so they land in the
crawler list too. Idle-time link preloads from pagerite.js carry an
``x-pagerite-preload`` header and are not tracked at all — the ping sent
when the user actually navigates does the counting.
Admin clients ping with ``hide=1``: the client record is flagged ``hide``,
``x-pagerite-preload`` header and are not tracked at all — the navigation
message sent when the user actually navigates does the counting.
Admin clients send ``hide``: the client record is flagged ``hide``,
which covers everything that client ever did — visits and crawler hits
from before the login included. Aggregates (site visits, page views,
transitions) are not stored; they are computed at display time from the
@@ -70,6 +75,27 @@ def _compact_user_agent(ua: str) -> str:
return " ".join(p for p in parts if p).strip()
class Ping(msgspec.Struct, omit_defaults=True):
"""One client message on the /_ws activity WebSocket.
Sent as a JSON text frame (msgspec-encoded, decoded to str for the
wire). ``to`` set: a navigation — internal page path or external https
exit URL. ``read`` alone (with ``fr``): an active reading-time update
for the page ``fr``; these arrive frequently while the user is active
and accumulate on the trail item. ``hide`` flags the client as an
admin: everything it ever did is excluded from the statistics.
"""
#: Path of the page the activity happened on ("" for the initial load).
fr: str = ""
#: Navigation target: internal path or external https exit URL.
to: str = ""
#: Active reading time (seconds) spent on ``fr`` since the last report.
read: int = 0
#: Admin client: record but hide everything from the statistics.
hide: bool = False
class Client(msgspec.Struct, omit_defaults=True):
"""Client metadata shared by visits, crawler hits and abuse hits.
@@ -151,7 +177,7 @@ class Visit(msgspec.Struct, omit_defaults=True):
class CrawlerHit(msgspec.Struct, omit_defaults=True):
"""A document GET that was never followed by an analytics ping.
"""A document GET that was never followed by an activity message.
Client metadata is held in ``Analytics.clients`` keyed by ``client``.
"""
@@ -392,19 +418,19 @@ class Store:
#: client hash -> index of the current visit in data.visits
self.sessions: dict[bytes, int] = {}
#: ip -> external https origin of the latest document GET carrying
#: one, stashed for the visit the client's initial ping starts.
#: one, stashed for the visit the client's initial message starts.
#: Internal or absent referers never touch the table.
self.pending_referers: dict[str, str] = {}
#: ip -> utm_* query parameters from the latest document GET that
#: carried any, stashed for the visit the client's initial ping starts.
#: carried any, stashed for the visit the client's initial message starts.
#: Only non-empty sets are stored, so a later parameter-less page
#: does not overwrite an earlier tagged landing URL.
self.pending_utms: dict[str, dict[str, str]] = {}
#: Document GETs that have not yet been matched by a ping. Kept
#: Document GETs that have not yet been matched by a message. Kept
#: in RAM only; expired entries are written to ``data.crawlers``.
self.pending_crawlers: list[CrawlerHit] = []
#: client hash -> {path: status} for recent document GETs, consumed
#: by the matching ping to record the status of each visited path.
#: by the matching message to record the status of each visited path.
self.pending_statuses: dict[bytes, dict[str, int]] = {}
#: ip -> number of plain (non-telltale) 404s seen, in RAM only;
#: reaching ``_ABUSE_404_THRESHOLD`` classifies the IP as abuse.
@@ -721,16 +747,16 @@ class Store:
) -> list[bytes]:
"""Stash the entry referer/UTM tags and queue a pending crawler hit.
Nothing is counted here — the client's initial /_a ping starts the
visit (only non-admin clients ping). Only a cross-origin https
Nothing is counted here — the client's first /_ws message starts the
visit (only non-admin clients report). Only a cross-origin https
referer updates the table; an internal or absent referer leaves any
stashed origin untouched. UTM parameters are kept only when the
landing URL actually carries them, so a subsequent parameter-less page
does not erase an earlier tagged landing.
Every document GET is also queued as a pending crawler hit. If a ping
from the same client arrives within ``_CRAWLER_TIMEOUT``, the hit is
discarded; otherwise it is flushed to ``data.crawlers``. The
Every document GET is also queued as a pending crawler hit. If a
message from the same client arrives within ``_CRAWLER_TIMEOUT``, the
hit is discarded; otherwise it is flushed to ``data.crawlers``. The
Accept-Language header is stored on the client record immediately;
host/geoip are filled in later by async enrichment.
@@ -794,15 +820,16 @@ class Store:
hide: bool = False,
read: int = 0,
) -> tuple[int | None, list[bytes]]:
"""Record a client navigation ping ({from, to, read} from pagerite.js).
"""Record a client activity message (``Ping`` from pagerite.js over /_ws).
``to`` is an internal path ("/...") or an https URL for exit links; a
missing/empty ``to`` means the page is being closed and only the
missing/empty ``to`` means a pure reading-time update and only the
``read`` time should be recorded. The transition is always counted when
``to`` is present; the trail only grows on first sight of a page within
the visit. ``read`` is the active time (seconds) spent on ``from_``.
the visit. ``read`` is the active time (seconds) spent on ``from_``
since the previous report.
A ping with no known session starts a fresh visit, consuming the
A message with no known session starts a fresh visit, consuming the
referer and UTM tags stashed by the document GET if there are any.
``hide`` is set by admin clients: the client record is flagged
@@ -811,7 +838,7 @@ class Store:
normally. Hidden clients are excluded from every statistic and list
at display time, and their pending crawler hits are discarded.
Pings from IPs classified as abuse, and pings whose User-Agent
Messages from IPs classified as abuse, and messages whose User-Agent
claims a JS-running crawler identity (``_is_bot_ua``), are ignored
entirely — the crawler's pending hits stay queued and flush to
``data.crawlers`` normally.
pagerite/app.py
+45 -38
View File
@@ -32,10 +32,10 @@ from xml.sax.saxutils import escape as xml_escape
import blake3
import httpx
import msgspec
from fastapi import (
FastAPI,
HTTPException,
Query,
Request,
WebSocket,
WebSocketDisconnect,
@@ -935,7 +935,7 @@ async def delete_page(path: str) -> None:
_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
def _client_ip(request: Request) -> str:
def _client_ip(request: Request | WebSocket) -> str:
"""Client IP: first X-Forwarded-For hop (we sit behind a proxy), else
the direct peer."""
forwarded = request.headers.get("x-forwarded-for", "").split(",")[0].strip()
@@ -1108,48 +1108,54 @@ async def analytics_page(request: Request) -> Response:
)
@app.post("/_a", status_code=204)
async def analytics_ping(
request: Request,
fr: str = Query(""),
to: str | None = Query(None),
hide: int = Query(0),
read: int = Query(0),
) -> None:
"""Record a navigation ping (?fr=&to=&hide=&read=); fire-and-forget.
@app.websocket("/_ws")
async def activity_ws(ws: WebSocket) -> None:
"""Collect visitor activity: navigations and reading-time updates.
The initial page-load ping carries only ``to``: the entry is attributed
to the referer/UTM tags stashed by the document GET (see _track_entry),
which JS cannot see once the page has loaded.
The reverse-DNS and DB-IP geoip lookups happen in a background task so
the response is never delayed by slow DNS or the first MMDB decompress.
Public, like the pages themselves (only /_api is gated); one connection
follows a browsing session. Messages are ``analytics.Ping`` structs as
JSON text frames; ``to`` set is a navigation, ``read`` alone a
reading-time update. The reverse-DNS and DB-IP geoip lookups happen in
background tasks so message handling is never delayed by slow DNS or
the first MMDB decompress.
"""
ip = _client_ip(request)
visit_index, flushed_clients = analytics_store.ping(
fr,
to,
ip,
request.headers.get("user-agent", ""),
request.headers.get("accept-language", ""),
hide=bool(hide),
read=read,
)
if visit_index is not None:
visit = analytics_store.data.visits[visit_index]
asyncio.create_task(_enrich_client(visit.client))
_schedule_client_enrichment(flushed_clients)
_schedule_favicon_fetch()
await ws.accept()
ip = _client_ip(ws)
ua = ws.headers.get("user-agent", "")
accept_language = ws.headers.get("accept-language", "")
try:
while True:
text = await ws.receive_text()
try:
msg = msgspec.json.decode(text.encode(), type=analytics.Ping)
except msgspec.DecodeError:
continue
visit_index, flushed_clients = analytics_store.ping(
msg.fr,
msg.to or None,
ip,
ua,
accept_language,
hide=msg.hide,
read=msg.read,
)
if visit_index is not None:
visit = analytics_store.data.visits[visit_index]
asyncio.create_task(_enrich_client(visit.client))
_schedule_client_enrichment(flushed_clients)
_schedule_favicon_fetch()
except WebSocketDisconnect:
pass
def _track_entry(path: str, request: Request, *, status: int = 200) -> list[bytes]:
"""Stash the referer/UTM tags and queue a pending crawler hit for the GET.
Nothing is counted on the GET itself — the client's /_a ping starts the
visit, so bots never register as visits (JS-running crawlers ping too,
but the ping handler ignores known bot UAs). (Admin clients ping too,
but with hide=1, which flags their visit hidden: it is recorded but
excluded from all statistics and from the crawler list.)
Nothing is counted on the GET itself — the client's first /_ws message
starts the visit, so bots never register as visits (JS-running crawlers
connect too, but the WebSocket handler ignores known bot UAs). (Admin
clients report too, but with hide, which flags their visit hidden: it is
recorded but excluded from all statistics and from the crawler list.)
The devserver's health probe (``GET /?from=devserver.py`` from
``127.0.0.1``) is ignored: it is not real traffic and would otherwise be
@@ -1161,7 +1167,8 @@ def _track_entry(path: str, request: Request, *, status: int = 200) -> list[byte
"""
if request.headers.get("x-pagerite-preload"):
# Idle-time page-cache warm-up by pagerite.js, not a page view: the
# ping sent when the user actually navigates does the counting.
# activity message sent when the user actually navigates does the
# counting.
# (Forging the header only hides a GET from the crawler stats; the
# path-based abuse classification is unaffected.)
return []