diff --git a/docs/analytics.md b/docs/analytics.md
index 6118957..91b51cc 100644
--- a/docs/analytics.md
+++ b/docs/analytics.md
@@ -54,6 +54,10 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with
tasks after the visit is stored, so the `/ _a` response is never delayed.
The decompressed `dbip-*.mmdb` file is kept in the repository root and
ignored by git.
+- **Crawler hits**: every document GET is queued in RAM as a pending crawler
+ hit. If a ping from the same (IP, User-Agent) pair arrives within 10
+ seconds the hit is discarded; otherwise it is written to `crawlers`.
+ Crawlers do not count as visits or views.
## Visits and sessions
@@ -81,8 +85,22 @@ Each `Visit` record:
`Accept-Language` region subtag, but overwritten by the DB-IP MMDB result
when a database is available,
- `ua` — raw `User-Agent` string from the initial ping,
+- `ua_pretty` — compact display form of the UA (browser/OS/device) when
+ parsable, otherwise the raw string,
- `utm` — `utm_*` query parameters from the landing URL, as a dict.
+Each `CrawlerHit` record:
+
+- `start` — timestamp of the document GET,
+- `entry` — page path requested,
+- `ip` — IP address,
+- `ua` — raw `User-Agent` header,
+- `ua_pretty` — compact display form of the UA when parsable,
+- `referer` — external https origin of the request, `""` for direct/none,
+- `query` — raw query string of the request.
+
+Crawler hits are grouped by User-Agent in the analytics viewer.
+
## Aggregates
- `transitions`: sparse nested dict `from -> to -> count`. `from` is the
diff --git a/frontend/src/AnalyticsView.vue b/frontend/src/AnalyticsView.vue
index 966c89d..7a67dc1 100644
--- a/frontend/src/AnalyticsView.vue
+++ b/frontend/src/AnalyticsView.vue
@@ -8,7 +8,14 @@
// See docs/analytics.md for the data format.
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
import { RANGES } from './analytics/time.js'
-import { calcTotalViews, formatVisitRows } from './analytics/format.js'
+import {
+ calcTotalViews,
+ copyIp,
+ countCrawlerUas,
+ formatCounts,
+ formatCrawlerRows,
+ formatVisitRows,
+} from './analytics/format.js'
import * as flagSvgs from 'country-flag-icons/string/3x2'
import TransitionGraph from './TransitionGraph.vue'
import VisitorCharts from './VisitorCharts.vue'
@@ -57,6 +64,9 @@ watch(range, (r) => {
})
const visitRows = computed(() => formatVisitRows(visits.value, pageTree.value))
+const crawlers = computed(() => data.value?.crawlers || [])
+const crawlerRows = computed(() => formatCrawlerRows(crawlers.value))
+const topCrawlerUas = computed(() => countCrawlerUas(crawlers.value).slice(0, 10))
function flagSvg(code) {
return flagSvgs[code?.toUpperCase()] || ''
@@ -106,7 +116,6 @@ function countryName(code) {
trail
referer
ip
-
host
lang
country
ua
@@ -123,14 +132,17 @@ function countryName(code) {
{{ v.referer }}
-
{{ v.ip }}
-
{{ v.host }}
+
+ {{ v.ipDisplay }}
+
{{ v.lang }}
—
-
{{ v.ua }}
+
{{ v.ua }}
{{ v.utm }}
@@ -138,6 +150,42 @@ function countryName(code) {
no visits recorded yet
+
+
+
Crawlers
+
+
top UAs: {{ formatCounts(topCrawlerUas) }}
+
+
+
+
+
+
when
+
entry
+
ip
+
ua
+
referer
+
query
+
+
+
+
+
{{ c.when }}
+
{{ c.entry }}
+
+ {{ c.ipDisplay }}
+
+
{{ c.ua }}
+
{{ c.referer }}
+
{{ c.query }}
+
+
+
+
+
no crawler hits recorded yet
+
@@ -268,6 +316,16 @@ function countryName(code) {
margin-left: 0.5rem;
}
+.visit-table .clickable-ip {
+ cursor: pointer;
+ text-decoration: underline;
+ text-decoration-style: dotted;
+}
+
+.visit-table .clickable-ip:hover {
+ color: var(--accent);
+}
+
.visit-table .ua {
max-width: 18rem;
overflow: hidden;
@@ -291,6 +349,15 @@ function countryName(code) {
display: block;
}
+.crawler-top-uas {
+ font-size: 0.9rem;
+ margin-bottom: 0.6rem;
+}
+
+.crawler-top-uas strong {
+ color: var(--muted);
+}
+
.empty, .loading, .error { color: var(--muted); }
.error { color: var(--error, #c00); }
diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js
index f322a92..0adcf76 100644
--- a/frontend/src/analytics/format.js
+++ b/frontend/src/analytics/format.js
@@ -3,6 +3,38 @@
* visit trail.
*/
+/**
+ * IPv4 unchanged, IPv6 returns the /64 network prefix in compact form.
+ * Falls back to the original value when parsing fails.
+ */
+export const hostIP = (ip) => {
+ try {
+ if (!ip || !ip.includes(':')) return ip
+ const strip = (s) => s.replace(/^\[|\]$/g, '')
+ const norm = strip(new URL(`http://[${ip}]/`).hostname)
+ const [l, r] = norm.split('::').map((s) => (s ? s.split(':') : []))
+ const full = r
+ ? [...l, ...Array(8 - l.length - r.length).fill('0'), ...r]
+ : l
+ return strip(
+ new URL(`http://[${full.slice(0, 4).join(':')}::]/`).hostname,
+ ).replace(/::$/, '')
+ } catch (e) {
+ console.error('hostIP processing failed for:', ip, e)
+ return ip
+ }
+}
+
+/** Copy the full IP to the clipboard, ignoring failures. */
+export async function copyIp(ip) {
+ if (!ip) return
+ try {
+ await navigator.clipboard.writeText(ip)
+ } catch {
+ /* ignore */
+ }
+}
+
/** Total page views across every page and every bucket. */
export function calcTotalViews(views) {
let n = 0
@@ -87,6 +119,37 @@ export function formatCounts(entries) {
return entries.map(([value, count]) => `${value} (${count})`).join(', ')
}
+/**
+ * Count distinct User-Agent strings among crawler hits, most common first.
+ * Returns an array of [ua, count] pairs.
+ */
+export function countCrawlerUas(crawlers) {
+ const counts = {}
+ for (const c of crawlers || []) {
+ const value = c.ua_pretty || c.ua || '(no UA)'
+ counts[value] = (counts[value] || 0) + 1
+ }
+ return Object.entries(counts).sort((a, b) => b[1] - a[1])
+}
+
+/**
+ * Format raw crawler hit records as rows for a technical table. Missing
+ * values become "—".
+ */
+export function formatCrawlerRows(crawlers) {
+ const dash = (s) => (s || '—')
+ return [...(crawlers || [])].reverse().map((c) => ({
+ when: new Date(c.start).toLocaleString(),
+ entry: dash(c.entry),
+ ip: c.ip || '',
+ ipDisplay: c.host || hostIP(c.ip) || c.ip || '—',
+ ua: c.ua_pretty || c.ua || '—',
+ uaRaw: c.ua || '',
+ referer: dash(c.referer),
+ query: dash(c.query),
+ }))
+}
+
/**
* Format raw visit records as rows for a technical table. Returns objects
* with display strings; missing values become "—". ``trail`` joins page
@@ -110,11 +173,13 @@ export function formatVisitRows(visits, pageTree) {
when: new Date(v.start).toLocaleString(),
trail,
referer: dash(v.referer),
- ip: dash(v.ip),
+ ip: v.ip || '',
+ ipDisplay: v.host || hostIP(v.ip) || v.ip || '—',
host: dash(v.host),
lang: dash(v.lang),
country: dash(v.country),
- ua: dash(v.ua),
+ ua: v.ua_pretty || v.ua || '—',
+ uaRaw: v.ua || '',
utm: utm || '—',
}
})
diff --git a/pagerite/analytics.py b/pagerite/analytics.py
index d481df6..614bf20 100644
--- a/pagerite/analytics.py
+++ b/pagerite/analytics.py
@@ -17,11 +17,34 @@ rewritten atomically on every recorded event.
import os
import re
import tempfile
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from urllib.parse import parse_qs, urlparse
import msgspec
+from ua_parser import parse
+
+
+def _compact_user_agent(ua: str) -> str:
+ """Format a User-Agent string into a compact display form.
+
+ Returns the original UA when the parser cannot identify the browser/OS.
+ """
+ if not ua or not ua.strip() or ua == "-":
+ return ""
+ r = parse(ua)
+ browser = r.user_agent.family if r.user_agent else None
+ ver = r.user_agent.major if r.user_agent else ""
+ os_name = r.os.family if r.os else None
+ dev = r.device.family if r.device else None
+ if browser in (None, "Other") and os_name in (None, "Other"):
+ return ua
+ browser = browser if browser and browser != "Other" else ""
+ os_name = os_name if os_name and os_name != "Other" else ""
+ if dev in (None, "Other") or dev == browser:
+ dev = ""
+ parts = [f"{browser}/{ver}" if browser else "", os_name, dev]
+ return " ".join(p for p in parts if p).strip()
class Visit(msgspec.Struct, omit_defaults=True):
@@ -47,15 +70,34 @@ class Visit(msgspec.Struct, omit_defaults=True):
country: str = ""
#: Raw User-Agent header from the initial ping.
ua: str = ""
+ #: Compact display form of ``ua`` (browser/OS/device) when parsable.
+ ua_pretty: str = ""
#: UTM query parameters from the landing URL, keyed by parameter name.
utm: dict[str, str] = {}
+class CrawlerHit(msgspec.Struct, omit_defaults=True):
+ """A document GET that was never followed by an analytics ping."""
+
+ start: datetime
+ entry: str
+ ip: str = ""
+ ua: str = ""
+ #: Compact display form of ``ua`` when parsable.
+ ua_pretty: str = ""
+ #: External https origin of the initial load, "" for direct/none.
+ referer: str = ""
+ #: Raw query string of the landing URL (UTM tags can be parsed from it).
+ query: str = ""
+
+
class Analytics(msgspec.Struct, omit_defaults=True):
"""Root of the analytics JSON file. Append-only by design: old data is
dropped by deleting list entries / bucket keys."""
visits: list[Visit] = []
+ #: Document GETs that never produced a ping, treated as crawler/bot hits.
+ crawlers: list[CrawlerHit] = []
#: Page transition matrix: from -> to -> count. ``from`` is the referer
#: origin or "(direct)" for initial loads, a page path for pings.
transitions: dict[str, dict[str, int]] = {}
@@ -125,6 +167,9 @@ def _utm_tags(query: str) -> dict[str, str]:
return {k: v[0] for k, v in parsed.items() if k.startswith("utm_")}
+_CRAWLER_TIMEOUT = timedelta(seconds=10)
+
+
class Store:
"""In-memory analytics data plus the (IP, UA) -> visit session map."""
@@ -147,6 +192,9 @@ class Store:
#: Only non-empty sets are stored, so a later parameter-less page
#: does not overwrite an earlier tagged landing URL.
self.pending_utms: dict[str, dict[str, str]] = {}
+ #: Document GETs that have not yet been matched by a ping. Kept
+ #: in RAM only; expired entries are written to ``data.crawlers``.
+ self.pending_crawlers: list[CrawlerHit] = []
def _save(self) -> None:
"""Rewrite the JSON file atomically (temp file + rename)."""
@@ -160,6 +208,21 @@ class Store:
except OSError:
pass # analytics must never break page serving
+ def _flush_crawlers(self, now: datetime | None = None) -> None:
+ """Move expired pending crawler hits into persistent ``data.crawlers``."""
+ if not self.pending_crawlers:
+ return
+ now = now or datetime.now(UTC)
+ cutoff = now - _CRAWLER_TIMEOUT
+ expired: list[CrawlerHit] = []
+ remaining: list[CrawlerHit] = []
+ for hit in self.pending_crawlers:
+ (expired if hit.start <= cutoff else remaining).append(hit)
+ if expired:
+ self.pending_crawlers = remaining
+ self.data.crawlers.extend(expired)
+ self._save()
+
def _count(self, table: dict[str, int], key: str) -> None:
table[key] = table.get(key, 0) + 1
@@ -183,6 +246,7 @@ class Store:
lang=lang,
country=country,
ua=ua,
+ ua_pretty=_compact_user_agent(ua),
utm=utm or {},
)
self.data.visits.append(visit)
@@ -215,10 +279,16 @@ class Store:
if changed:
self._save()
- def entry_referer(
- self, referer: str, own_origin: str, ip: str, query: str = ""
+ def track_entry(
+ self,
+ referer: str,
+ own_origin: str,
+ ip: str,
+ ua: str,
+ entry: str,
+ query: str = "",
) -> None:
- """Stash the entry referer and UTM tags of a document GET for ping attribution.
+ """Stash the entry referer/UTM tags and queue a pending crawler hit.
Nothing is counted here — the client's initial /_a ping starts the
visit (only non-admin clients ping). Only a cross-origin https
@@ -226,7 +296,13 @@ class Store:
stashed origin untouched. UTM parameters are kept only when the
landing URL actually carries them, so a subsequent parameter-less page
does not erase an earlier tagged landing.
+
+ Every document GET is also queued as a pending crawler hit. If a ping
+ from the same (IP, UA) pair arrives within ``_CRAWLER_TIMEOUT``, the
+ hit is discarded; otherwise it is flushed to ``data.crawlers``.
"""
+ now = datetime.now(UTC)
+ self._flush_crawlers(now)
if referer:
origin = _origin(referer)
if origin is not None and origin != own_origin:
@@ -234,6 +310,17 @@ class Store:
utms = _utm_tags(query)
if utms:
self.pending_utms[ip] = utms
+ self.pending_crawlers.append(
+ CrawlerHit(
+ start=now,
+ entry=entry,
+ ip=ip,
+ ua=ua,
+ ua_pretty=_compact_user_agent(ua),
+ referer=self.pending_referers.get(ip, ""),
+ query=query,
+ )
+ )
def ping(
self,
@@ -254,6 +341,12 @@ class Store:
Returns the index of the new visit when one is created, so callers
can enrich it later with non-blocking lookups (host, geoip country).
"""
+ self._flush_crawlers()
+ # A real visitor ping cancels any pending crawler hits from this
+ # (IP, UA) pair.
+ self.pending_crawlers = [
+ hit for hit in self.pending_crawlers if not (hit.ip == ip and hit.ua == ua)
+ ]
if to.startswith("/") and not to.startswith("//"):
target = _internal_path(to) or ""
else:
diff --git a/pagerite/app.py b/pagerite/app.py
index 01dfdac..9f3487b 100644
--- a/pagerite/app.py
+++ b/pagerite/app.py
@@ -651,16 +651,18 @@ async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
def _track_entry(path: str, request: Request) -> None:
- """Stash the referer and UTM tags of the document GET for the initial ping.
+ """Stash the referer/UTM tags and queue a pending crawler hit for the GET.
Nothing is counted on the GET itself — the client's /_a ping starts the
- visit, so bots and admin browsing never register.
+ visit, so bots and admin browsing never register as visits.
"""
own_origin = f"https://{urlparse(str(request.base_url)).netloc}"
- analytics_store.entry_referer(
+ analytics_store.track_entry(
request.headers.get("referer", ""),
own_origin,
_client_ip(request),
+ request.headers.get("user-agent", ""),
+ "/" if path == "" else f"/{path}",
str(request.url.query),
)
@@ -680,6 +682,15 @@ def _is_reserved(path: str) -> bool:
return any(not _SLUG_RE.match(seg) for seg in path.split("/"))
+def _is_trackable_path(path: str) -> bool:
+ """Content URLs only: skip auth endpoints and reserved/machinery paths."""
+ if not path:
+ return True
+ if path == "auth" or path.startswith("auth/"):
+ return False
+ return not _is_reserved(path)
+
+
def _check_reserved(path: str) -> None:
"""Reject paths that do not follow the slug charset."""
if _is_reserved(path):
@@ -881,7 +892,8 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
etag = f'"{path}@{node.modified.timestamp()}v{data.version}"'
if request.headers.get("if-none-match") == etag:
return Response(status_code=304)
- _track_entry(path, request)
+ if _is_trackable_path(path):
+ _track_entry(path, request)
return HTMLResponse(
views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, str(request.base_url).rstrip("/")),
headers={
@@ -893,7 +905,8 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
if node is not None and node.published and node.content is None:
# Category label without a landing page: placeholder with the pen
# to create it (404 — no page here, but the node is real).
- _track_entry(path, request)
+ if _is_trackable_path(path):
+ _track_entry(path, request)
return HTMLResponse(
views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html),
404,
@@ -908,5 +921,6 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
for slug, item in sorted_nodes(data.menu):
if item.published:
return RedirectResponse(f"/{slug}")
- _track_entry(path, request)
+ if _is_trackable_path(path):
+ _track_entry(path, request)
return HTMLResponse(views.render_not_found(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html), 404)
diff --git a/pyproject.toml b/pyproject.toml
index c69e12d..02168d9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ dependencies = [
"maxminddb>=3.1.1",
"mdit-py-plugins>=0.6.1",
"pygments>=2.20.0",
+ "ua-parser>=1.0.2",
]
[project.scripts]