analytics: add crawler tracking, pretty UA/IP display and copy-to-clipboard

docs/analytics.md

@@ -54,6 +54,10 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with
   tasks after the visit is stored, so the `/ _a` response is never delayed.
   The decompressed `dbip-*.mmdb` file is kept in the repository root and
   ignored by git.
+- **Crawler hits**: every document GET is queued in RAM as a pending crawler
+  hit.  If a ping from the same (IP, User-Agent) pair arrives within 10
+  seconds the hit is discarded; otherwise it is written to `crawlers`.
+  Crawlers do not count as visits or views.
 
 ## Visits and sessions
 
@@ -81,8 +85,22 @@ Each `Visit` record:
   `Accept-Language` region subtag, but overwritten by the DB-IP MMDB result
   when a database is available,
 - `ua` — raw `User-Agent` string from the initial ping,
+- `ua_pretty` — compact display form of the UA (browser/OS/device) when
+  parsable, otherwise the raw string,
 - `utm` — `utm_*` query parameters from the landing URL, as a dict.
 
+Each `CrawlerHit` record:
+
+- `start` — timestamp of the document GET,
+- `entry` — page path requested,
+- `ip` — IP address,
+- `ua` — raw `User-Agent` header,
+- `ua_pretty` — compact display form of the UA when parsable,
+- `referer` — external https origin of the request, `""` for direct/none,
+- `query` — raw query string of the request.
+
+Crawler hits are grouped by User-Agent in the analytics viewer.
+
 ## Aggregates
 
 - `transitions`: sparse nested dict `from -> to -> count`. `from` is the

frontend/src/AnalyticsView.vue

@@ -8,7 +8,14 @@
 // See docs/analytics.md for the data format.
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
 import { RANGES } from './analytics/time.js'
-import { calcTotalViews, formatVisitRows } from './analytics/format.js'
+import {
+  calcTotalViews,
+  copyIp,
+  countCrawlerUas,
+  formatCounts,
+  formatCrawlerRows,
+  formatVisitRows,
+} from './analytics/format.js'
 import * as flagSvgs from 'country-flag-icons/string/3x2'
 import TransitionGraph from './TransitionGraph.vue'
 import VisitorCharts from './VisitorCharts.vue'
@@ -57,6 +64,9 @@ watch(range, (r) => {
 })
 
 const visitRows = computed(() => formatVisitRows(visits.value, pageTree.value))
+const crawlers = computed(() => data.value?.crawlers || [])
+const crawlerRows = computed(() => formatCrawlerRows(crawlers.value))
+const topCrawlerUas = computed(() => countCrawlerUas(crawlers.value).slice(0, 10))
 
 function flagSvg(code) {
   return flagSvgs[code?.toUpperCase()] || ''
@@ -106,7 +116,6 @@ function countryName(code) {
                   <th>trail</th>
                   <th>referer</th>
                   <th>ip</th>
-                  <th>host</th>
                   <th>lang</th>
                   <th>country</th>
                   <th>ua</th>
@@ -123,14 +132,17 @@ function countryName(code) {
                     </a>
                   </td>
                   <td>{{ v.referer }}</td>
-                  <td>{{ v.ip }}</td>
+                  <td>
-                  <td>{{ v.host }}</td>
+                    <span class="clickable-ip"
+                          :title="`Click to copy full IP: ${v.ip}`"
+                          @click="copyIp(v.ip)">{{ v.ipDisplay }}</span>
+                  </td>
                   <td>{{ v.lang }}</td>
                   <td class="country">
                     <span v-if="flagSvg(v.country)" class="flag" v-html="flagSvg(v.country)" :title="countryName(v.country) || v.country"></span>
                     <template v-else>—</template>
                   </td>
-                  <td class="ua" :title="v.ua">{{ v.ua }}</td>
+                  <td class="ua" :title="v.uaRaw">{{ v.ua }}</td>
                   <td>{{ v.utm }}</td>
                 </tr>
               </tbody>
@@ -138,6 +150,42 @@ function countryName(code) {
           </div>
           <p v-else class="empty">no visits recorded yet</p>
         </section>
+
+        <section>
+          <h2>Crawlers</h2>
+          <div v-if="topCrawlerUas.length" class="crawler-top-uas">
+            <p><strong>top UAs:</strong> {{ formatCounts(topCrawlerUas) }}</p>
+          </div>
+          <div v-if="crawlerRows.length" class="visit-table-wrap">
+            <table class="visit-table">
+              <thead>
+                <tr>
+                  <th>when</th>
+                  <th>entry</th>
+                  <th>ip</th>
+                  <th>ua</th>
+                  <th>referer</th>
+                  <th>query</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr v-for="(c, i) in crawlerRows" :key="i">
+                  <td class="when">{{ c.when }}</td>
+                  <td>{{ c.entry }}</td>
+                  <td>
+                    <span class="clickable-ip"
+                          :title="`Click to copy full IP: ${c.ip}`"
+                          @click="copyIp(c.ip)">{{ c.ipDisplay }}</span>
+                  </td>
+                  <td class="ua" :title="c.uaRaw">{{ c.ua }}</td>
+                  <td>{{ c.referer }}</td>
+                  <td>{{ c.query }}</td>
+                </tr>
+              </tbody>
+            </table>
+          </div>
+          <p v-else class="empty">no crawler hits recorded yet</p>
+        </section>
       </template>
     </div>
   </div>
@@ -268,6 +316,16 @@ function countryName(code) {
   margin-left: 0.5rem;
 }
 
+.visit-table .clickable-ip {
+  cursor: pointer;
+  text-decoration: underline;
+  text-decoration-style: dotted;
+}
+
+.visit-table .clickable-ip:hover {
+  color: var(--accent);
+}
+
 .visit-table .ua {
   max-width: 18rem;
   overflow: hidden;
@@ -291,6 +349,15 @@ function countryName(code) {
   display: block;
 }
 
+.crawler-top-uas {
+  font-size: 0.9rem;
+  margin-bottom: 0.6rem;
+}
+
+.crawler-top-uas strong {
+  color: var(--muted);
+}
+
 .empty, .loading, .error { color: var(--muted); }
 .error { color: var(--error, #c00); }
 </style>

frontend/src/analytics/format.js

@@ -3,6 +3,38 @@
  * visit trail.
  */
 
+/**
+ * IPv4 unchanged, IPv6 returns the /64 network prefix in compact form.
+ * Falls back to the original value when parsing fails.
+ */
+export const hostIP = (ip) => {
+  try {
+    if (!ip || !ip.includes(':')) return ip
+    const strip = (s) => s.replace(/^\[|\]$/g, '')
+    const norm = strip(new URL(`http://[${ip}]/`).hostname)
+    const [l, r] = norm.split('::').map((s) => (s ? s.split(':') : []))
+    const full = r
+      ? [...l, ...Array(8 - l.length - r.length).fill('0'), ...r]
+      : l
+    return strip(
+      new URL(`http://[${full.slice(0, 4).join(':')}::]/`).hostname,
+    ).replace(/::$/, '')
+  } catch (e) {
+    console.error('hostIP processing failed for:', ip, e)
+    return ip
+  }
+}
+
+/** Copy the full IP to the clipboard, ignoring failures. */
+export async function copyIp(ip) {
+  if (!ip) return
+  try {
+    await navigator.clipboard.writeText(ip)
+  } catch {
+    /* ignore */
+  }
+}
+
 /** Total page views across every page and every bucket. */
 export function calcTotalViews(views) {
   let n = 0
@@ -87,6 +119,37 @@ export function formatCounts(entries) {
   return entries.map(([value, count]) => `${value} (${count})`).join(', ')
 }
 
+/**
+ * Count distinct User-Agent strings among crawler hits, most common first.
+ * Returns an array of [ua, count] pairs.
+ */
+export function countCrawlerUas(crawlers) {
+  const counts = {}
+  for (const c of crawlers || []) {
+    const value = c.ua_pretty || c.ua || '(no UA)'
+    counts[value] = (counts[value] || 0) + 1
+  }
+  return Object.entries(counts).sort((a, b) => b[1] - a[1])
+}
+
+/**
+ * Format raw crawler hit records as rows for a technical table.  Missing
+ * values become "—".
+ */
+export function formatCrawlerRows(crawlers) {
+  const dash = (s) => (s || '—')
+  return [...(crawlers || [])].reverse().map((c) => ({
+    when: new Date(c.start).toLocaleString(),
+    entry: dash(c.entry),
+    ip: c.ip || '',
+    ipDisplay: c.host || hostIP(c.ip) || c.ip || '—',
+    ua: c.ua_pretty || c.ua || '—',
+    uaRaw: c.ua || '',
+    referer: dash(c.referer),
+    query: dash(c.query),
+  }))
+}
+
 /**
  * Format raw visit records as rows for a technical table.  Returns objects
  * with display strings; missing values become "—".  ``trail`` joins page
@@ -110,11 +173,13 @@ export function formatVisitRows(visits, pageTree) {
       when: new Date(v.start).toLocaleString(),
       trail,
       referer: dash(v.referer),
-      ip: dash(v.ip),
+      ip: v.ip || '',
+      ipDisplay: v.host || hostIP(v.ip) || v.ip || '—',
       host: dash(v.host),
       lang: dash(v.lang),
       country: dash(v.country),
-      ua: dash(v.ua),
+      ua: v.ua_pretty || v.ua || '—',
+      uaRaw: v.ua || '',
       utm: utm || '—',
     }
   })

pagerite/analytics.py

@@ -17,11 +17,34 @@ rewritten atomically on every recorded event.
 import os
 import re
 import tempfile
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from urllib.parse import parse_qs, urlparse
 
 import msgspec
+from ua_parser import parse
+
+
+def _compact_user_agent(ua: str) -> str:
+    """Format a User-Agent string into a compact display form.
+
+    Returns the original UA when the parser cannot identify the browser/OS.
+    """
+    if not ua or not ua.strip() or ua == "-":
+        return ""
+    r = parse(ua)
+    browser = r.user_agent.family if r.user_agent else None
+    ver = r.user_agent.major if r.user_agent else ""
+    os_name = r.os.family if r.os else None
+    dev = r.device.family if r.device else None
+    if browser in (None, "Other") and os_name in (None, "Other"):
+        return ua
+    browser = browser if browser and browser != "Other" else ""
+    os_name = os_name if os_name and os_name != "Other" else ""
+    if dev in (None, "Other") or dev == browser:
+        dev = ""
+    parts = [f"{browser}/{ver}" if browser else "", os_name, dev]
+    return " ".join(p for p in parts if p).strip()
 
 
 class Visit(msgspec.Struct, omit_defaults=True):
@@ -47,15 +70,34 @@ class Visit(msgspec.Struct, omit_defaults=True):
     country: str = ""
     #: Raw User-Agent header from the initial ping.
     ua: str = ""
+    #: Compact display form of ``ua`` (browser/OS/device) when parsable.
+    ua_pretty: str = ""
     #: UTM query parameters from the landing URL, keyed by parameter name.
     utm: dict[str, str] = {}
 
 
+class CrawlerHit(msgspec.Struct, omit_defaults=True):
+    """A document GET that was never followed by an analytics ping."""
+
+    start: datetime
+    entry: str
+    ip: str = ""
+    ua: str = ""
+    #: Compact display form of ``ua`` when parsable.
+    ua_pretty: str = ""
+    #: External https origin of the initial load, "" for direct/none.
+    referer: str = ""
+    #: Raw query string of the landing URL (UTM tags can be parsed from it).
+    query: str = ""
+
+
 class Analytics(msgspec.Struct, omit_defaults=True):
     """Root of the analytics JSON file. Append-only by design: old data is
     dropped by deleting list entries / bucket keys."""
 
     visits: list[Visit] = []
+    #: Document GETs that never produced a ping, treated as crawler/bot hits.
+    crawlers: list[CrawlerHit] = []
     #: Page transition matrix: from -> to -> count. ``from`` is the referer
     #: origin or "(direct)" for initial loads, a page path for pings.
     transitions: dict[str, dict[str, int]] = {}
@@ -125,6 +167,9 @@ def _utm_tags(query: str) -> dict[str, str]:
     return {k: v[0] for k, v in parsed.items() if k.startswith("utm_")}
 
 
+_CRAWLER_TIMEOUT = timedelta(seconds=10)
+
+
 class Store:
     """In-memory analytics data plus the (IP, UA) -> visit session map."""
 
@@ -147,6 +192,9 @@ class Store:
         #: Only non-empty sets are stored, so a later parameter-less page
         #: does not overwrite an earlier tagged landing URL.
         self.pending_utms: dict[str, dict[str, str]] = {}
+        #: Document GETs that have not yet been matched by a ping.  Kept
+        #: in RAM only; expired entries are written to ``data.crawlers``.
+        self.pending_crawlers: list[CrawlerHit] = []
 
     def _save(self) -> None:
         """Rewrite the JSON file atomically (temp file + rename)."""
@@ -160,6 +208,21 @@ class Store:
         except OSError:
             pass  # analytics must never break page serving
 
+    def _flush_crawlers(self, now: datetime | None = None) -> None:
+        """Move expired pending crawler hits into persistent ``data.crawlers``."""
+        if not self.pending_crawlers:
+            return
+        now = now or datetime.now(UTC)
+        cutoff = now - _CRAWLER_TIMEOUT
+        expired: list[CrawlerHit] = []
+        remaining: list[CrawlerHit] = []
+        for hit in self.pending_crawlers:
+            (expired if hit.start <= cutoff else remaining).append(hit)
+        if expired:
+            self.pending_crawlers = remaining
+            self.data.crawlers.extend(expired)
+            self._save()
+
     def _count(self, table: dict[str, int], key: str) -> None:
         table[key] = table.get(key, 0) + 1
 
@@ -183,6 +246,7 @@ class Store:
             lang=lang,
             country=country,
             ua=ua,
+            ua_pretty=_compact_user_agent(ua),
             utm=utm or {},
         )
         self.data.visits.append(visit)
@@ -215,10 +279,16 @@ class Store:
         if changed:
             self._save()
 
-    def entry_referer(
+    def track_entry(
-        self, referer: str, own_origin: str, ip: str, query: str = ""
+        self,
+        referer: str,
+        own_origin: str,
+        ip: str,
+        ua: str,
+        entry: str,
+        query: str = "",
     ) -> None:
-        """Stash the entry referer and UTM tags of a document GET for ping attribution.
+        """Stash the entry referer/UTM tags and queue a pending crawler hit.
 
         Nothing is counted here — the client's initial /_a ping starts the
         visit (only non-admin clients ping). Only a cross-origin https
@@ -226,7 +296,13 @@ class Store:
         stashed origin untouched.  UTM parameters are kept only when the
         landing URL actually carries them, so a subsequent parameter-less page
         does not erase an earlier tagged landing.
+
+        Every document GET is also queued as a pending crawler hit.  If a ping
+        from the same (IP, UA) pair arrives within ``_CRAWLER_TIMEOUT``, the
+        hit is discarded; otherwise it is flushed to ``data.crawlers``.
         """
+        now = datetime.now(UTC)
+        self._flush_crawlers(now)
         if referer:
             origin = _origin(referer)
             if origin is not None and origin != own_origin:
@@ -234,6 +310,17 @@ class Store:
         utms = _utm_tags(query)
         if utms:
             self.pending_utms[ip] = utms
+        self.pending_crawlers.append(
+            CrawlerHit(
+                start=now,
+                entry=entry,
+                ip=ip,
+                ua=ua,
+                ua_pretty=_compact_user_agent(ua),
+                referer=self.pending_referers.get(ip, ""),
+                query=query,
+            )
+        )
 
     def ping(
         self,
@@ -254,6 +341,12 @@ class Store:
         Returns the index of the new visit when one is created, so callers
         can enrich it later with non-blocking lookups (host, geoip country).
         """
+        self._flush_crawlers()
+        # A real visitor ping cancels any pending crawler hits from this
+        # (IP, UA) pair.
+        self.pending_crawlers = [
+            hit for hit in self.pending_crawlers if not (hit.ip == ip and hit.ua == ua)
+        ]
         if to.startswith("/") and not to.startswith("//"):
             target = _internal_path(to) or ""
         else:

pagerite/app.py

@@ -651,16 +651,18 @@ async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
 
 
 def _track_entry(path: str, request: Request) -> None:
-    """Stash the referer and UTM tags of the document GET for the initial ping.
+    """Stash the referer/UTM tags and queue a pending crawler hit for the GET.
 
     Nothing is counted on the GET itself — the client's /_a ping starts the
-    visit, so bots and admin browsing never register.
+    visit, so bots and admin browsing never register as visits.
     """
     own_origin = f"https://{urlparse(str(request.base_url)).netloc}"
-    analytics_store.entry_referer(
+    analytics_store.track_entry(
         request.headers.get("referer", ""),
         own_origin,
         _client_ip(request),
+        request.headers.get("user-agent", ""),
+        "/" if path == "" else f"/{path}",
         str(request.url.query),
     )
 
@@ -680,6 +682,15 @@ def _is_reserved(path: str) -> bool:
     return any(not _SLUG_RE.match(seg) for seg in path.split("/"))
 
 
+def _is_trackable_path(path: str) -> bool:
+    """Content URLs only: skip auth endpoints and reserved/machinery paths."""
+    if not path:
+        return True
+    if path == "auth" or path.startswith("auth/"):
+        return False
+    return not _is_reserved(path)
+
+
 def _check_reserved(path: str) -> None:
     """Reject paths that do not follow the slug charset."""
     if _is_reserved(path):
@@ -881,7 +892,8 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
         etag = f'"{path}@{node.modified.timestamp()}v{data.version}"'
         if request.headers.get("if-none-match") == etag:
             return Response(status_code=304)
-        _track_entry(path, request)
+        if _is_trackable_path(path):
+            _track_entry(path, request)
         return HTMLResponse(
             views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, str(request.base_url).rstrip("/")),
             headers={
@@ -893,7 +905,8 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
     if node is not None and node.published and node.content is None:
         # Category label without a landing page: placeholder with the pen
         # to create it (404 — no page here, but the node is real).
-        _track_entry(path, request)
+        if _is_trackable_path(path):
+            _track_entry(path, request)
         return HTMLResponse(
             views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html),
             404,
@@ -908,5 +921,6 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
         for slug, item in sorted_nodes(data.menu):
             if item.published:
                 return RedirectResponse(f"/{slug}")
-    _track_entry(path, request)
+    if _is_trackable_path(path):
+        _track_entry(path, request)
     return HTMLResponse(views.render_not_found(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html), 404)

pyproject.toml

@@ -25,6 +25,7 @@ dependencies = [
     "maxminddb>=3.1.1",
     "mdit-py-plugins>=0.6.1",
     "pygments>=2.20.0",
+    "ua-parser>=1.0.2",
 ]
 
 [project.scripts]