Record response status per path; mark 404 trails red in the viewer

Document GETs now stash their status (200/404) in a pending table,
consumed by the matching ping: visits gain a per-path statuses map and
crawler hits a status field. Trail links with a 404 status render in
red with the status code in the tooltip, alongside the read time.

docs/analytics.md

@@ -147,6 +147,8 @@ Each `Visit` record:
   does not append.
 - `utm` — `utm_*` query parameters from the landing URL, as a dict.
 - `read` — active reading time per path (seconds), keyed by path.
+- `statuses` — HTTP status of the response when each path was first seen
+  (200 or 404), keyed by path.
 
 Each `CrawlerHit` record:
 
@@ -154,7 +156,9 @@ Each `CrawlerHit` record:
 - `entry` — page path requested,
 - `client` — 6-byte blake3 hash referencing `Analytics.clients`,
 - `referer` — external https origin of the request, `""` for direct/none,
-- `query` — raw query string of the request.
+- `query` — raw query string of the request,
+- `status` — HTTP status of the served response (200 for a real page, 404
+  for a category placeholder or missing page).
 
 Each `AbuseHit` record:
 
@@ -173,6 +177,10 @@ that triggered classification are lifted to the top, followed by other 404s
 and then document GETs from the abuser.  Within each category paths are
 sorted by count descending, then by their earliest hit.
 
+In the visitor and crawler tables, internal paths that returned a 404 status
+are shown in red and the link title includes the status code, so it is easy
+to tell misses from real pages at a glance.
+
 ## Aggregates
 
 - `transitions`: time series of page transitions, sparse nested dict

frontend/src/AnalyticsView.vue

@@ -425,6 +425,11 @@ const abuseRows = computed(() => formatAbuseRows(rangeData.value?.abuse || [], c
   margin-left: 0.5rem;
 }
 
+.analytics-view :deep(.trail-link.error),
+.analytics-view :deep(.trail-link.error:hover) {
+  color: var(--error, #c00);
+}
+
 .visit-table .utm-tag {
   display: inline-block;
   max-width: 100%;

frontend/src/TrailLink.vue

@@ -1,18 +1,33 @@
 <script setup>
-import { formatCount } from './analytics/format.js'
+import { computed } from 'vue'
+import { formatCount, formatReadTime } from './analytics/format.js'
 
-defineProps({
+const props = defineProps({
   step: { type: Object, required: true },
   count: { type: Number, default: 0 },
 })
 
 defineEmits(['close'])
+
+const hasError = computed(() => props.step.status >= 400)
+
+const title = computed(() => {
+  const parts = [props.step.title]
+  if (props.step.readSeconds > 0) {
+    parts.push(formatReadTime(props.step.readSeconds))
+  }
+  if (hasError.value) {
+    parts.push(`${props.step.status}`)
+  }
+  return parts.filter(Boolean).join(' — ')
+})
 </script>
 
 <template>
   <a class="trail-link"
+     :class="{ error: hasError }"
      :href="step.path"
-     :title="count > 1 ? `${step.title} (${count} hits)` : step.title"
+     :title="title"
      :target="step.external ? '_blank' : undefined"
      :rel="step.external ? 'noopener' : undefined"
      @click="(e) => { if (!step.external) $emit('close') }">

frontend/src/analytics/format.js

@@ -240,6 +240,14 @@ export function formatWhenIso(ts) {
   return `${new Date(ts).toISOString().split('.')[0]}Z`
 }
 
+/**
+ * Compact read time for tooltips: "50s" under a minute, "1m23s" otherwise.
+ */
+export function formatReadTime(seconds) {
+  if (seconds < 60) return `${seconds}s`
+  return `${Math.floor(seconds / 60)}m${seconds % 60}s`
+}
+
 /**
  * Compact visitor counts: plain below 1k, then 1.2k / 10k / 1.2M.
  * Truncated, not rounded.
@@ -359,13 +367,16 @@ export function formatCrawlerRows(crawlers, clients, pageTree, now = Date.now())
     const start = new Date(c.start).getTime()
     if (start > g.lastStart) g.lastStart = start
     if (c.entry?.startsWith('/')) {
-      g.pages.set(c.entry, (g.pages.get(c.entry) || 0) + 1)
+      const existing = g.pages.get(c.entry) || { count: 0, status: c.status || 200 }
+      existing.count += 1
+      if (c.status != null) existing.status = c.status
+      g.pages.set(c.entry, existing)
     }
     groups.set(c.client, g)
   }
   const totalHits = (g) => {
     let n = 0
-    for (const c of g.pages.values()) n += c
+    for (const p of g.pages.values()) n += p.count
     return n
   }
   return [...groups.values()]
@@ -380,8 +391,8 @@ export function formatCrawlerRows(crawlers, clients, pageTree, now = Date.now())
         lastSeenIso: formatWhenIso(g.lastStart),
         lastSeenLocal: formatWhenLocal(g.lastStart),
         pages: [...g.pages.entries()]
-          .sort((a, b) => b[1] - a[1])
+          .sort((a, b) => b[1].count - a[1].count)
-          .map(([path, count]) => ({ ...stepOf(path, titles), count })),
+          .map(([path, info]) => ({ ...stepOf(path, titles), count: info.count, status: info.status })),
         ip: client.ip || '',
         ipDisplay: isHost ? mainDomain(host) : hostIP(client.ip) || client.ip || '—',
         isHost,
@@ -499,8 +510,17 @@ export function formatVisitRows(visits, clients, pageTree, now = Date.now()) {
   const titles = buildTitleMap(pageTree)
   return [...(visits || [])].reverse().slice(0, 20).map((v) => {
     const client = (clients || {})[v.client] || {}
+    const read = v.read || {}
+    const statuses = v.statuses || {}
     const trail = [v.entry, ...(v.trail || [])]
-      .map((p) => stepOf(p, titles))
+      .map((p) => {
+        const step = stepOf(p, titles)
+        if (step) {
+          if (read[p]) step.readSeconds = read[p]
+          if (statuses[p]) step.status = statuses[p]
+        }
+        return step
+      })
       .filter(Boolean)
     const utmKeys = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content']
     const utmValues = utmKeys.map((k) => (v.utm || {})[k]).filter(Boolean)

pagerite/analytics.py

@@ -109,6 +109,8 @@ class Visit(msgspec.Struct, omit_defaults=True):
     utm: dict[str, str] = {}
     #: Active reading time per path (seconds), keyed by path.
     read: dict[str, int] = {}
+    #: HTTP status of the response when the path was first seen (200 or 404).
+    statuses: dict[str, int] = {}
 
 
 class CrawlerHit(msgspec.Struct, omit_defaults=True):
@@ -125,6 +127,8 @@ class CrawlerHit(msgspec.Struct, omit_defaults=True):
     referer: str = ""
     #: Raw query string of the landing URL (UTM tags can be parsed from it).
     query: str = ""
+    #: HTTP status of the served response (200 or 404 for content pages).
+    status: int = 200
 
 
 class AbuseHit(msgspec.Struct, omit_defaults=True):
@@ -324,6 +328,9 @@ class Store:
         #: Document GETs that have not yet been matched by a ping.  Kept
         #: in RAM only; expired entries are written to ``data.crawlers``.
         self.pending_crawlers: list[CrawlerHit] = []
+        #: client hash -> {path: status} for recent document GETs, consumed
+        #: by the matching ping to record the status of each visited path.
+        self.pending_statuses: dict[bytes, dict[str, int]] = {}
         #: ip -> number of plain (non-telltale) 404s seen, in RAM only;
         #: reaching ``_ABUSE_404_THRESHOLD`` classifies the IP as abuse.
         self.not_found_counts: dict[str, int] = {}
@@ -571,6 +578,7 @@ class Store:
         referer: str,
         client_hash: bytes,
         utm: dict[str, str] | None = None,
+        status: int = 200,
     ) -> Visit:
         now = datetime.now(UTC)
         visit = Visit(
@@ -580,6 +588,7 @@ class Store:
             client=client_hash,
             utm=utm or {},
         )
+        visit.statuses[entry] = status
         self.data.visits.append(visit)
         self.sessions[client_hash] = len(self.data.visits) - 1
         self._count(self.data.site_visits, _bucket(now))
@@ -595,6 +604,8 @@ class Store:
         ua: str,
         full_path: str,
         accept_language: str = "",
+        *,
+        status: int = 200,
     ) -> list[bytes]:
         """Stash the entry referer/UTM tags and queue a pending crawler hit.
 
@@ -642,8 +653,10 @@ class Store:
                 client=client_hash,
                 referer=self.pending_referers.get(ip, ""),
                 query=query,
+                status=status,
             )
         )
+        self.pending_statuses.setdefault(client_hash, {})[entry] = status
         return flushed
 
     def _add_read(self, client_hash: bytes, path: str, seconds: int) -> None:
@@ -699,6 +712,7 @@ class Store:
             self.pending_crawlers = [
                 hit for hit in self.pending_crawlers if hit.client != client_hash
             ]
+            self.pending_statuses.pop(client_hash, None)
             index = self.sessions.pop(client_hash, None)
             if index is not None and index < len(self.data.visits):
                 self._remove_visit(index)
@@ -730,6 +744,10 @@ class Store:
             return None, flushed
         index = self.sessions.get(client_hash)
         fr = fr_path or "(direct)"
+        statuses = self.pending_statuses.setdefault(client_hash, {})
+        target_status = statuses.pop(target, None) or 200
+        if not statuses:
+            self.pending_statuses.pop(client_hash, None)
         if index is None or index >= len(self.data.visits):
             # No known session: the initial ping of a fresh page load (or
             # missing data after a server restart) — start a visit.
@@ -740,6 +758,7 @@ class Store:
                 self.pending_referers.pop(ip, ""),
                 client_hash,
                 utm=self.pending_utms.pop(ip, {}),
+                status=target_status,
             )
         else:
             visit = self.data.visits[index]
@@ -750,6 +769,7 @@ class Store:
             # First-seen only: repeat pages and repeated exits don't append.
             if visit.entry != target and target not in visit.trail:
                 visit.trail.append(target)
+            visit.statuses[target] = target_status
         self._save()
         visit_index = index if index is not None and index < len(self.data.visits) else None
         return visit_index, flushed

pagerite/app.py

@@ -853,7 +853,7 @@ async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
     _schedule_client_enrichment(flushed_clients)
 
 
-def _track_entry(path: str, request: Request) -> list[bytes]:
+def _track_entry(path: str, request: Request, *, status: int = 200) -> list[bytes]:
     """Stash the referer/UTM tags and queue a pending crawler hit for the GET.
 
     Nothing is counted on the GET itself — the client's /_a ping starts the
@@ -890,6 +890,7 @@ def _track_entry(path: str, request: Request) -> list[bytes]:
         request.headers.get("user-agent", ""),
         full_path,
         request.headers.get("accept-language", ""),
+        status=status,
     )
 
 
@@ -1228,7 +1229,7 @@ async def show_page(request: Request, path: str) -> Response:
         # Category label without a landing page: placeholder with the pen
         # to create it (404 — no page here, but the node is real).
         if _is_trackable_path(path):
-            flushed = _track_entry(path, request)
+            flushed = _track_entry(path, request, status=404)
             _schedule_client_enrichment(flushed)
         return _html_response(
             request,
@@ -1254,6 +1255,6 @@ async def show_page(request: Request, path: str) -> Response:
             accept_language,
         )
         asyncio.create_task(_enrich_client(client_hash))
-        flushed = _track_entry(path, request)
+        flushed = _track_entry(path, request, status=404)
         _schedule_client_enrichment(flushed)
     return _html_response(request, "not-found", path, 404)