diff --git a/docs/analytics.md b/docs/analytics.md
index 7ca688a..d39b6cd 100644
--- a/docs/analytics.md
+++ b/docs/analytics.md
@@ -147,6 +147,8 @@ Each `Visit` record:
does not append.
- `utm` — `utm_*` query parameters from the landing URL, as a dict.
- `read` — active reading time per path (seconds), keyed by path.
+- `statuses` — HTTP status of the response when each path was first seen
+ (200 or 404), keyed by path.
Each `CrawlerHit` record:
@@ -154,7 +156,9 @@ Each `CrawlerHit` record:
- `entry` — page path requested,
- `client` — 6-byte blake3 hash referencing `Analytics.clients`,
- `referer` — external https origin of the request, `""` for direct/none,
-- `query` — raw query string of the request.
+- `query` — raw query string of the request,
+- `status` — HTTP status of the served response (200 for a real page, 404
+ for a category placeholder or missing page).
Each `AbuseHit` record:
@@ -173,6 +177,10 @@ that triggered classification are lifted to the top, followed by other 404s
and then document GETs from the abuser. Within each category paths are
sorted by count descending, then by their earliest hit.
+In the visitor and crawler tables, internal paths that returned a 404 status
+are shown in red and the link title includes the status code, so it is easy
+to tell misses from real pages at a glance.
+
## Aggregates
- `transitions`: time series of page transitions, sparse nested dict
diff --git a/frontend/src/AnalyticsView.vue b/frontend/src/AnalyticsView.vue
index 8d885ad..4a11576 100644
--- a/frontend/src/AnalyticsView.vue
+++ b/frontend/src/AnalyticsView.vue
@@ -425,6 +425,11 @@ const abuseRows = computed(() => formatAbuseRows(rangeData.value?.abuse || [], c
margin-left: 0.5rem;
}
+.analytics-view :deep(.trail-link.error),
+.analytics-view :deep(.trail-link.error:hover) {
+ color: var(--error, #c00);
+}
+
.visit-table .utm-tag {
display: inline-block;
max-width: 100%;
diff --git a/frontend/src/TrailLink.vue b/frontend/src/TrailLink.vue
index a9482e1..3fceaa2 100644
--- a/frontend/src/TrailLink.vue
+++ b/frontend/src/TrailLink.vue
@@ -1,18 +1,33 @@
{ if (!step.external) $emit('close') }">
diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js
index d66fc9d..5a0f846 100644
--- a/frontend/src/analytics/format.js
+++ b/frontend/src/analytics/format.js
@@ -240,6 +240,14 @@ export function formatWhenIso(ts) {
return `${new Date(ts).toISOString().split('.')[0]}Z`
}
+/**
+ * Compact read time for tooltips: "50s" under a minute, "1m23s" otherwise.
+ */
+export function formatReadTime(seconds) {
+ if (seconds < 60) return `${seconds}s`
+ return `${Math.floor(seconds / 60)}m${seconds % 60}s`
+}
+
/**
* Compact visitor counts: plain below 1k, then 1.2k / 10k / 1.2M.
* Truncated, not rounded.
@@ -359,13 +367,16 @@ export function formatCrawlerRows(crawlers, clients, pageTree, now = Date.now())
const start = new Date(c.start).getTime()
if (start > g.lastStart) g.lastStart = start
if (c.entry?.startsWith('/')) {
- g.pages.set(c.entry, (g.pages.get(c.entry) || 0) + 1)
+ const existing = g.pages.get(c.entry) || { count: 0, status: c.status || 200 }
+ existing.count += 1
+ if (c.status != null) existing.status = c.status
+ g.pages.set(c.entry, existing)
}
groups.set(c.client, g)
}
const totalHits = (g) => {
let n = 0
- for (const c of g.pages.values()) n += c
+ for (const p of g.pages.values()) n += p.count
return n
}
return [...groups.values()]
@@ -380,8 +391,8 @@ export function formatCrawlerRows(crawlers, clients, pageTree, now = Date.now())
lastSeenIso: formatWhenIso(g.lastStart),
lastSeenLocal: formatWhenLocal(g.lastStart),
pages: [...g.pages.entries()]
- .sort((a, b) => b[1] - a[1])
- .map(([path, count]) => ({ ...stepOf(path, titles), count })),
+ .sort((a, b) => b[1].count - a[1].count)
+ .map(([path, info]) => ({ ...stepOf(path, titles), count: info.count, status: info.status })),
ip: client.ip || '',
ipDisplay: isHost ? mainDomain(host) : hostIP(client.ip) || client.ip || '—',
isHost,
@@ -499,8 +510,17 @@ export function formatVisitRows(visits, clients, pageTree, now = Date.now()) {
const titles = buildTitleMap(pageTree)
return [...(visits || [])].reverse().slice(0, 20).map((v) => {
const client = (clients || {})[v.client] || {}
+ const read = v.read || {}
+ const statuses = v.statuses || {}
const trail = [v.entry, ...(v.trail || [])]
- .map((p) => stepOf(p, titles))
+ .map((p) => {
+ const step = stepOf(p, titles)
+ if (step) {
+ if (read[p]) step.readSeconds = read[p]
+ if (statuses[p]) step.status = statuses[p]
+ }
+ return step
+ })
.filter(Boolean)
const utmKeys = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content']
const utmValues = utmKeys.map((k) => (v.utm || {})[k]).filter(Boolean)
diff --git a/pagerite/analytics.py b/pagerite/analytics.py
index fac9c5c..819e766 100644
--- a/pagerite/analytics.py
+++ b/pagerite/analytics.py
@@ -109,6 +109,8 @@ class Visit(msgspec.Struct, omit_defaults=True):
utm: dict[str, str] = {}
#: Active reading time per path (seconds), keyed by path.
read: dict[str, int] = {}
+ #: HTTP status of the response when the path was first seen (200 or 404).
+ statuses: dict[str, int] = {}
class CrawlerHit(msgspec.Struct, omit_defaults=True):
@@ -125,6 +127,8 @@ class CrawlerHit(msgspec.Struct, omit_defaults=True):
referer: str = ""
#: Raw query string of the landing URL (UTM tags can be parsed from it).
query: str = ""
+ #: HTTP status of the served response (200 or 404 for content pages).
+ status: int = 200
class AbuseHit(msgspec.Struct, omit_defaults=True):
@@ -324,6 +328,9 @@ class Store:
#: Document GETs that have not yet been matched by a ping. Kept
#: in RAM only; expired entries are written to ``data.crawlers``.
self.pending_crawlers: list[CrawlerHit] = []
+ #: client hash -> {path: status} for recent document GETs, consumed
+ #: by the matching ping to record the status of each visited path.
+ self.pending_statuses: dict[bytes, dict[str, int]] = {}
#: ip -> number of plain (non-telltale) 404s seen, in RAM only;
#: reaching ``_ABUSE_404_THRESHOLD`` classifies the IP as abuse.
self.not_found_counts: dict[str, int] = {}
@@ -571,6 +578,7 @@ class Store:
referer: str,
client_hash: bytes,
utm: dict[str, str] | None = None,
+ status: int = 200,
) -> Visit:
now = datetime.now(UTC)
visit = Visit(
@@ -580,6 +588,7 @@ class Store:
client=client_hash,
utm=utm or {},
)
+ visit.statuses[entry] = status
self.data.visits.append(visit)
self.sessions[client_hash] = len(self.data.visits) - 1
self._count(self.data.site_visits, _bucket(now))
@@ -595,6 +604,8 @@ class Store:
ua: str,
full_path: str,
accept_language: str = "",
+ *,
+ status: int = 200,
) -> list[bytes]:
"""Stash the entry referer/UTM tags and queue a pending crawler hit.
@@ -642,8 +653,10 @@ class Store:
client=client_hash,
referer=self.pending_referers.get(ip, ""),
query=query,
+ status=status,
)
)
+ self.pending_statuses.setdefault(client_hash, {})[entry] = status
return flushed
def _add_read(self, client_hash: bytes, path: str, seconds: int) -> None:
@@ -699,6 +712,7 @@ class Store:
self.pending_crawlers = [
hit for hit in self.pending_crawlers if hit.client != client_hash
]
+ self.pending_statuses.pop(client_hash, None)
index = self.sessions.pop(client_hash, None)
if index is not None and index < len(self.data.visits):
self._remove_visit(index)
@@ -730,6 +744,10 @@ class Store:
return None, flushed
index = self.sessions.get(client_hash)
fr = fr_path or "(direct)"
+ statuses = self.pending_statuses.setdefault(client_hash, {})
+ target_status = statuses.pop(target, None) or 200
+ if not statuses:
+ self.pending_statuses.pop(client_hash, None)
if index is None or index >= len(self.data.visits):
# No known session: the initial ping of a fresh page load (or
# missing data after a server restart) — start a visit.
@@ -740,6 +758,7 @@ class Store:
self.pending_referers.pop(ip, ""),
client_hash,
utm=self.pending_utms.pop(ip, {}),
+ status=target_status,
)
else:
visit = self.data.visits[index]
@@ -750,6 +769,7 @@ class Store:
# First-seen only: repeat pages and repeated exits don't append.
if visit.entry != target and target not in visit.trail:
visit.trail.append(target)
+ visit.statuses[target] = target_status
self._save()
visit_index = index if index is not None and index < len(self.data.visits) else None
return visit_index, flushed
diff --git a/pagerite/app.py b/pagerite/app.py
index 95f22e8..c67eb9c 100644
--- a/pagerite/app.py
+++ b/pagerite/app.py
@@ -853,7 +853,7 @@ async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
_schedule_client_enrichment(flushed_clients)
-def _track_entry(path: str, request: Request) -> list[bytes]:
+def _track_entry(path: str, request: Request, *, status: int = 200) -> list[bytes]:
"""Stash the referer/UTM tags and queue a pending crawler hit for the GET.
Nothing is counted on the GET itself — the client's /_a ping starts the
@@ -890,6 +890,7 @@ def _track_entry(path: str, request: Request) -> list[bytes]:
request.headers.get("user-agent", ""),
full_path,
request.headers.get("accept-language", ""),
+ status=status,
)
@@ -1228,7 +1229,7 @@ async def show_page(request: Request, path: str) -> Response:
# Category label without a landing page: placeholder with the pen
# to create it (404 — no page here, but the node is real).
if _is_trackable_path(path):
- flushed = _track_entry(path, request)
+ flushed = _track_entry(path, request, status=404)
_schedule_client_enrichment(flushed)
return _html_response(
request,
@@ -1254,6 +1255,6 @@ async def show_page(request: Request, path: str) -> Response:
accept_language,
)
asyncio.create_task(_enrich_client(client_hash))
- flushed = _track_entry(path, request)
+ flushed = _track_entry(path, request, status=404)
_schedule_client_enrichment(flushed)
return _html_response(request, "not-found", path, 404)