LeoVasanko/pagerite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

analytics: add crawler tracking, pretty UA/IP display and copy-to-clipboard

Browse Source
This commit is contained in:
LeoVasanko
2026-08-20 20:31:04 +00:00
parent 1a1a21712e
commit 7556bb7f4f
6 changed files with 275 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/analytics.md
+18
View File
@@ -54,6 +54,10 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with
tasks after the visit is stored, so the `/ _a` response is never delayed.
The decompressed `dbip-*.mmdb` file is kept in the repository root and
ignored by git.
- **Crawler hits**: every document GET is queued in RAM as a pending crawler
hit. If a ping from the same (IP, User-Agent) pair arrives within 10
seconds the hit is discarded; otherwise it is written to `crawlers`.
Crawlers do not count as visits or views.
## Visits and sessions
@@ -81,8 +85,22 @@ Each `Visit` record:
`Accept-Language` region subtag, but overwritten by the DB-IP MMDB result
when a database is available,
- `ua` — raw `User-Agent` string from the initial ping,
- `ua_pretty` — compact display form of the UA (browser/OS/device) when
parsable, otherwise the raw string,
- `utm` — `utm_*` query parameters from the landing URL, as a dict.
Each `CrawlerHit` record:
- `start` — timestamp of the document GET,
- `entry` — page path requested,
- `ip` — IP address,
- `ua` — raw `User-Agent` header,
- `ua_pretty` — compact display form of the UA when parsable,
- `referer` — external https origin of the request, `""` for direct/none,
- `query` — raw query string of the request.
Crawler hits are grouped by User-Agent in the analytics viewer.
## Aggregates
- `transitions`: sparse nested dict `from -> to -> count`. `from` is the
frontend/src/AnalyticsView.vue
+72 -5
View File
@@ -8,7 +8,14 @@
// See docs/analytics.md for the data format.
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
import { RANGES } from './analytics/time.js'
import { calcTotalViews, formatVisitRows } from './analytics/format.js'
import {
calcTotalViews,
copyIp,
countCrawlerUas,
formatCounts,
formatCrawlerRows,
formatVisitRows,
} from './analytics/format.js'
import * as flagSvgs from 'country-flag-icons/string/3x2'
import TransitionGraph from './TransitionGraph.vue'
import VisitorCharts from './VisitorCharts.vue'
@@ -57,6 +64,9 @@ watch(range, (r) => {
})
const visitRows = computed(() => formatVisitRows(visits.value, pageTree.value))
const crawlers = computed(() => data.value?.crawlers || [])
const crawlerRows = computed(() => formatCrawlerRows(crawlers.value))
const topCrawlerUas = computed(() => countCrawlerUas(crawlers.value).slice(0, 10))
function flagSvg(code) {
return flagSvgs[code?.toUpperCase()] || ''
@@ -106,7 +116,6 @@ function countryName(code) {
<th>trail</th>
<th>referer</th>
<th>ip</th>
<th>host</th>
<th>lang</th>
<th>country</th>
<th>ua</th>
@@ -123,14 +132,17 @@ function countryName(code) {
</a>
</td>
<td>{{ v.referer }}</td>
<td>{{ v.ip }}</td>
<td>{{ v.host }}</td>
<td>
<span class="clickable-ip"
:title="`Click to copy full IP: ${v.ip}`"
@click="copyIp(v.ip)">{{ v.ipDisplay }}</span>
</td>
<td>{{ v.lang }}</td>
<td class="country">
<span v-if="flagSvg(v.country)" class="flag" v-html="flagSvg(v.country)" :title="countryName(v.country) || v.country"></span>
<template v-else>—</template>
</td>
<td class="ua" :title="v.ua">{{ v.ua }}</td>
<td class="ua" :title="v.uaRaw">{{ v.ua }}</td>
<td>{{ v.utm }}</td>
</tr>
</tbody>
@@ -138,6 +150,42 @@ function countryName(code) {
</div>
<p v-else class="empty">no visits recorded yet</p>
</section>
<section>
<h2>Crawlers</h2>
<div v-if="topCrawlerUas.length" class="crawler-top-uas">
<p><strong>top UAs:</strong> {{ formatCounts(topCrawlerUas) }}</p>
</div>
<div v-if="crawlerRows.length" class="visit-table-wrap">
<table class="visit-table">
<thead>
<tr>
<th>when</th>
<th>entry</th>
<th>ip</th>
<th>ua</th>
<th>referer</th>
<th>query</th>
</tr>
</thead>
<tbody>
<tr v-for="(c, i) in crawlerRows" :key="i">
<td class="when">{{ c.when }}</td>
<td>{{ c.entry }}</td>
<td>
<span class="clickable-ip"
:title="`Click to copy full IP: ${c.ip}`"
@click="copyIp(c.ip)">{{ c.ipDisplay }}</span>
</td>
<td class="ua" :title="c.uaRaw">{{ c.ua }}</td>
<td>{{ c.referer }}</td>
<td>{{ c.query }}</td>
</tr>
</tbody>
</table>
</div>
<p v-else class="empty">no crawler hits recorded yet</p>
</section>
</template>
</div>
</div>
@@ -268,6 +316,16 @@ function countryName(code) {
margin-left: 0.5rem;
}
.visit-table .clickable-ip {
cursor: pointer;
text-decoration: underline;
text-decoration-style: dotted;
}
.visit-table .clickable-ip:hover {
color: var(--accent);
}
.visit-table .ua {
max-width: 18rem;
overflow: hidden;
@@ -291,6 +349,15 @@ function countryName(code) {
display: block;
}
.crawler-top-uas {
font-size: 0.9rem;
margin-bottom: 0.6rem;
}
.crawler-top-uas strong {
color: var(--muted);
}
.empty, .loading, .error { color: var(--muted); }
.error { color: var(--error, #c00); }
</style>
frontend/src/analytics/format.js
+67 -2
View File
@@ -3,6 +3,38 @@
* visit trail.
*/
/**
* IPv4 unchanged, IPv6 returns the /64 network prefix in compact form.
* Falls back to the original value when parsing fails.
*/
export const hostIP = (ip) => {
try {
if (!ip || !ip.includes(':')) return ip
const strip = (s) => s.replace(/^\[|\]$/g, '')
const norm = strip(new URL(`http://[${ip}]/`).hostname)
const [l, r] = norm.split('::').map((s) => (s ? s.split(':') : []))
const full = r
? [...l, ...Array(8 - l.length - r.length).fill('0'), ...r]
: l
return strip(
new URL(`http://[${full.slice(0, 4).join(':')}::]/`).hostname,
).replace(/::$/, '')
} catch (e) {
console.error('hostIP processing failed for:', ip, e)
return ip
}
}
/** Copy the full IP to the clipboard, ignoring failures. */
export async function copyIp(ip) {
if (!ip) return
try {
await navigator.clipboard.writeText(ip)
} catch {
/* ignore */
}
}
/** Total page views across every page and every bucket. */
export function calcTotalViews(views) {
let n = 0
@@ -87,6 +119,37 @@ export function formatCounts(entries) {
return entries.map(([value, count]) => `${value} (${count})`).join(', ')
}
/**
* Count distinct User-Agent strings among crawler hits, most common first.
* Returns an array of [ua, count] pairs.
*/
export function countCrawlerUas(crawlers) {
const counts = {}
for (const c of crawlers || []) {
const value = c.ua_pretty || c.ua || '(no UA)'
counts[value] = (counts[value] || 0) + 1
}
return Object.entries(counts).sort((a, b) => b[1] - a[1])
}
/**
* Format raw crawler hit records as rows for a technical table. Missing
* values become "—".
*/
export function formatCrawlerRows(crawlers) {
const dash = (s) => (s || '—')
return [...(crawlers || [])].reverse().map((c) => ({
when: new Date(c.start).toLocaleString(),
entry: dash(c.entry),
ip: c.ip || '',
ipDisplay: c.host || hostIP(c.ip) || c.ip || '—',
ua: c.ua_pretty || c.ua || '—',
uaRaw: c.ua || '',
referer: dash(c.referer),
query: dash(c.query),
}))
}
/**
* Format raw visit records as rows for a technical table. Returns objects
* with display strings; missing values become "—". ``trail`` joins page
@@ -110,11 +173,13 @@ export function formatVisitRows(visits, pageTree) {
when: new Date(v.start).toLocaleString(),
trail,
referer: dash(v.referer),
ip: dash(v.ip),
ip: v.ip || '',
ipDisplay: v.host || hostIP(v.ip) || v.ip || '—',
host: dash(v.host),
lang: dash(v.lang),
country: dash(v.country),
ua: dash(v.ua),
ua: v.ua_pretty || v.ua || '—',
uaRaw: v.ua || '',
utm: utm || '—',
}
})
pagerite/analytics.py
+97 -4
View File
@@ -17,11 +17,34 @@ rewritten atomically on every recorded event.
import os
import re
import tempfile
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from pathlib import Path
from urllib.parse import parse_qs, urlparse
import msgspec
from ua_parser import parse
def _compact_user_agent(ua: str) -> str:
"""Format a User-Agent string into a compact display form.
Returns the original UA when the parser cannot identify the browser/OS.
"""
if not ua or not ua.strip() or ua == "-":
return ""
r = parse(ua)
browser = r.user_agent.family if r.user_agent else None
ver = r.user_agent.major if r.user_agent else ""
os_name = r.os.family if r.os else None
dev = r.device.family if r.device else None
if browser in (None, "Other") and os_name in (None, "Other"):
return ua
browser = browser if browser and browser != "Other" else ""
os_name = os_name if os_name and os_name != "Other" else ""
if dev in (None, "Other") or dev == browser:
dev = ""
parts = [f"{browser}/{ver}" if browser else "", os_name, dev]
return " ".join(p for p in parts if p).strip()
class Visit(msgspec.Struct, omit_defaults=True):
@@ -47,15 +70,34 @@ class Visit(msgspec.Struct, omit_defaults=True):
country: str = ""
#: Raw User-Agent header from the initial ping.
ua: str = ""
#: Compact display form of ``ua`` (browser/OS/device) when parsable.
ua_pretty: str = ""
#: UTM query parameters from the landing URL, keyed by parameter name.
utm: dict[str, str] = {}
class CrawlerHit(msgspec.Struct, omit_defaults=True):
"""A document GET that was never followed by an analytics ping."""
start: datetime
entry: str
ip: str = ""
ua: str = ""
#: Compact display form of ``ua`` when parsable.
ua_pretty: str = ""
#: External https origin of the initial load, "" for direct/none.
referer: str = ""
#: Raw query string of the landing URL (UTM tags can be parsed from it).
query: str = ""
class Analytics(msgspec.Struct, omit_defaults=True):
"""Root of the analytics JSON file. Append-only by design: old data is
dropped by deleting list entries / bucket keys."""
visits: list[Visit] = []
#: Document GETs that never produced a ping, treated as crawler/bot hits.
crawlers: list[CrawlerHit] = []
#: Page transition matrix: from -> to -> count. ``from`` is the referer
#: origin or "(direct)" for initial loads, a page path for pings.
transitions: dict[str, dict[str, int]] = {}
@@ -125,6 +167,9 @@ def _utm_tags(query: str) -> dict[str, str]:
return {k: v[0] for k, v in parsed.items() if k.startswith("utm_")}
_CRAWLER_TIMEOUT = timedelta(seconds=10)
class Store:
"""In-memory analytics data plus the (IP, UA) -> visit session map."""
@@ -147,6 +192,9 @@ class Store:
#: Only non-empty sets are stored, so a later parameter-less page
#: does not overwrite an earlier tagged landing URL.
self.pending_utms: dict[str, dict[str, str]] = {}
#: Document GETs that have not yet been matched by a ping. Kept
#: in RAM only; expired entries are written to ``data.crawlers``.
self.pending_crawlers: list[CrawlerHit] = []
def _save(self) -> None:
"""Rewrite the JSON file atomically (temp file + rename)."""
@@ -160,6 +208,21 @@ class Store:
except OSError:
pass # analytics must never break page serving
def _flush_crawlers(self, now: datetime | None = None) -> None:
"""Move expired pending crawler hits into persistent ``data.crawlers``."""
if not self.pending_crawlers:
return
now = now or datetime.now(UTC)
cutoff = now - _CRAWLER_TIMEOUT
expired: list[CrawlerHit] = []
remaining: list[CrawlerHit] = []
for hit in self.pending_crawlers:
(expired if hit.start <= cutoff else remaining).append(hit)
if expired:
self.pending_crawlers = remaining
self.data.crawlers.extend(expired)
self._save()
def _count(self, table: dict[str, int], key: str) -> None:
table[key] = table.get(key, 0) + 1
@@ -183,6 +246,7 @@ class Store:
lang=lang,
country=country,
ua=ua,
ua_pretty=_compact_user_agent(ua),
utm=utm or {},
)
self.data.visits.append(visit)
@@ -215,10 +279,16 @@ class Store:
if changed:
self._save()
def entry_referer(
self, referer: str, own_origin: str, ip: str, query: str = ""
def track_entry(
self,
referer: str,
own_origin: str,
ip: str,
ua: str,
entry: str,
query: str = "",
) -> None:
"""Stash the entry referer and UTM tags of a document GET for ping attribution.
"""Stash the entry referer/UTM tags and queue a pending crawler hit.
Nothing is counted here — the client's initial /_a ping starts the
visit (only non-admin clients ping). Only a cross-origin https
@@ -226,7 +296,13 @@ class Store:
stashed origin untouched. UTM parameters are kept only when the
landing URL actually carries them, so a subsequent parameter-less page
does not erase an earlier tagged landing.
Every document GET is also queued as a pending crawler hit. If a ping
from the same (IP, UA) pair arrives within ``_CRAWLER_TIMEOUT``, the
hit is discarded; otherwise it is flushed to ``data.crawlers``.
"""
now = datetime.now(UTC)
self._flush_crawlers(now)
if referer:
origin = _origin(referer)
if origin is not None and origin != own_origin:
@@ -234,6 +310,17 @@ class Store:
utms = _utm_tags(query)
if utms:
self.pending_utms[ip] = utms
self.pending_crawlers.append(
CrawlerHit(
start=now,
entry=entry,
ip=ip,
ua=ua,
ua_pretty=_compact_user_agent(ua),
referer=self.pending_referers.get(ip, ""),
query=query,
)
)
def ping(
self,
@@ -254,6 +341,12 @@ class Store:
Returns the index of the new visit when one is created, so callers
can enrich it later with non-blocking lookups (host, geoip country).
"""
self._flush_crawlers()
# A real visitor ping cancels any pending crawler hits from this
# (IP, UA) pair.
self.pending_crawlers = [
hit for hit in self.pending_crawlers if not (hit.ip == ip and hit.ua == ua)
]
if to.startswith("/") and not to.startswith("//"):
target = _internal_path(to) or ""
else:
pagerite/app.py
+20 -6
View File
@@ -651,16 +651,18 @@ async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
def _track_entry(path: str, request: Request) -> None:
"""Stash the referer and UTM tags of the document GET for the initial ping.
"""Stash the referer/UTM tags and queue a pending crawler hit for the GET.
Nothing is counted on the GET itself — the client's /_a ping starts the
visit, so bots and admin browsing never register.
visit, so bots and admin browsing never register as visits.
"""
own_origin = f"https://{urlparse(str(request.base_url)).netloc}"
analytics_store.entry_referer(
analytics_store.track_entry(
request.headers.get("referer", ""),
own_origin,
_client_ip(request),
request.headers.get("user-agent", ""),
"/" if path == "" else f"/{path}",
str(request.url.query),
)
@@ -680,6 +682,15 @@ def _is_reserved(path: str) -> bool:
return any(not _SLUG_RE.match(seg) for seg in path.split("/"))
def _is_trackable_path(path: str) -> bool:
"""Content URLs only: skip auth endpoints and reserved/machinery paths."""
if not path:
return True
if path == "auth" or path.startswith("auth/"):
return False
return not _is_reserved(path)
def _check_reserved(path: str) -> None:
"""Reject paths that do not follow the slug charset."""
if _is_reserved(path):
@@ -881,7 +892,8 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
etag = f'"{path}@{node.modified.timestamp()}v{data.version}"'
if request.headers.get("if-none-match") == etag:
return Response(status_code=304)
_track_entry(path, request)
if _is_trackable_path(path):
_track_entry(path, request)
return HTMLResponse(
views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, str(request.base_url).rstrip("/")),
headers={
@@ -893,7 +905,8 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
if node is not None and node.published and node.content is None:
# Category label without a landing page: placeholder with the pen
# to create it (404 — no page here, but the node is real).
_track_entry(path, request)
if _is_trackable_path(path):
_track_entry(path, request)
return HTMLResponse(
views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html),
404,
@@ -908,5 +921,6 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
for slug, item in sorted_nodes(data.menu):
if item.published:
return RedirectResponse(f"/{slug}")
_track_entry(path, request)
if _is_trackable_path(path):
_track_entry(path, request)
return HTMLResponse(views.render_not_found(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html), 404)
pyproject.toml
+1
View File
@@ -25,6 +25,7 @@ dependencies = [
"maxminddb>=3.1.1",
"mdit-py-plugins>=0.6.1",
"pygments>=2.20.0",
"ua-parser>=1.0.2",
]
[project.scripts]