+
● {{ s.label }}
@@ -76,7 +97,7 @@ const viewChart = computed(() => buildChart(viewSeries.value))
/* The svg is stretched (preserveAspectRatio none), so all text lives in
HTML overlays positioned by the same fractions the geometry uses. */
.chartwrap {
- padding-left: 2.2rem; /* y labels */
+ padding-left: 2.8rem; /* y labels */
}
.plot {
@@ -103,8 +124,8 @@ const viewChart = computed(() => buildChart(viewSeries.value))
.ylab {
position: absolute;
- left: -2.2rem;
- width: 1.9rem;
+ left: -2.8rem;
+ width: 2.6rem;
text-align: right;
transform: translateY(50%);
font-size: 0.7rem;
@@ -154,6 +175,11 @@ const viewChart = computed(() => buildChart(viewSeries.value))
opacity: 0.15;
}
+.chart .bar {
+ fill: var(--accent);
+ opacity: 0.15;
+}
+
.chart .line {
fill: none;
stroke: var(--accent);
@@ -176,10 +202,11 @@ const viewChart = computed(() => buildChart(viewSeries.value))
.yaxis-label {
position: absolute;
top: 50%;
- left: -2.2rem;
+ left: -2.8rem;
font-size: 0.7rem;
color: var(--muted);
writing-mode: vertical-rl;
+ white-space: nowrap;
transform: translateY(-50%) rotate(180deg);
}
diff --git a/frontend/src/analytics/chart.js b/frontend/src/analytics/chart.js
index 6d3db8c..461906e 100644
--- a/frontend/src/analytics/chart.js
+++ b/frontend/src/analytics/chart.js
@@ -5,7 +5,7 @@
* rates (hour on the week view, day on month+).
*/
-import { DAY, HOUR, WEEK, mondayUTC } from './time.js'
+import { DAY, HOUR, MIN5, WEEK, mondayUTC } from './time.js'
export const CHART_W = 720
export const CHART_H = 180
@@ -206,9 +206,10 @@ export function spline(pts) {
}
/** Build a full chart model from a series descriptor produced by time.js. */
-export function buildChart(input) {
+export function buildChart(input, now = Date.now()) {
if (!input || !input.series.length) return null
- const { series, t0, t1, rate, binMinutes, unitMinutes } = input
+ if (input.unit === '5min') return buildDayChart(input, now)
+ const { series, t0, t1, rate, binMinutes, unitMinutes, unit } = input
// Values are per-unit rates (hour on the week view, day on month+); the
// y max is derived from the *smoothed* curves so random single-bucket
// spikes don't blow up the scale. Smoothing works on raw counts (its edge
@@ -283,7 +284,91 @@ export function buildChart(input) {
x: x(t), left: ((t - t0) / (t1 - t0)) * 100,
label: fmtTick(t, t1 - t0), line: true,
}))
- return { max, majors, minors, series: drawn, xticks }
+ return { max, majors, minors, series: drawn, xticks, unit }
+}
+
+/**
+ * Day view: 5-minute bars for the last 24 hours. Bars are drawn at raw
+ * counts; the skyline uses a projected full-bucket value for the still-open
+ * final bucket. The y scale is derived from the projected skyline maximum.
+ */
+export function buildDayChart(input, now = Date.now()) {
+ const { series, t0, t1 } = input
+ const points = series[0]?.points || []
+ const n = points.length
+ if (!n) return null
+ const bucketMs = (t1 - t0) / n
+ const bucketWidth = CHART_W / n
+ const gap = 0.2
+ const barWidth = Math.max(0.2, bucketWidth - gap)
+
+ const x = (i) => i * bucketWidth + gap / 2
+ const prevRaw = n > 1 ? points[n - 2].count : 0
+ const projected = points.map((p, i) => {
+ if (i !== n - 1) return p.count
+ const bucketStart = t0 + i * bucketMs
+ const elapsed = Math.max(1, Math.min(bucketMs, now - bucketStart))
+ // Blend the observed partial bucket with the previous full bucket:
+ // the longer the current bucket has run, the less we borrow from it.
+ const share = elapsed / bucketMs
+ return p.count + prevRaw * (1 - share)
+ })
+ const highest = Math.max(0, ...projected)
+ const { max, step, minor } = yScale(highest)
+ const y = (v) => PAD_TOP + (1 - Math.max(0, v) / max) * (CHART_H - PAD_TOP)
+
+ const bars = points.map((p, i) => {
+ const bx = x(i)
+ const by = y(p.count)
+ return {
+ x: bx,
+ y: by,
+ width: barWidth,
+ height: CHART_H - by,
+ raw: p.count,
+ projected: projected[i],
+ }
+ })
+
+ let skyline = ''
+ for (let i = 0; i < bars.length; i++) {
+ const b = bars[i]
+ const top = y(b.projected)
+ if (i === 0) {
+ skyline += `M${b.x},${top} H${b.x + b.width}`
+ } else {
+ skyline += ` V${top} H${b.x + b.width}`
+ }
+ }
+
+ const majors = []
+ const minors = []
+ const nMajor = Math.round(max / step)
+ for (let k = 0; k <= nMajor; k++) {
+ const v = k * step
+ majors.push({ value: v, y: y(v), bottom: (1 - PAD_TOP / CHART_H) * (v / max) * 100 })
+ }
+ if (minor) {
+ for (let v = minor; v < max; v += minor) {
+ if (v % step !== 0) minors.push({ y: y(v) })
+ }
+ }
+
+ const xticks = []
+ const tickStep = 3 * HOUR
+ const firstTick = Math.ceil(t0 / tickStep) * tickStep
+ for (let t = firstTick; t < t1; t += tickStep) {
+ if (t < t0) continue
+ const d = new Date(t)
+ xticks.push({
+ x: ((t - t0) / (t1 - t0)) * CHART_W,
+ left: ((t - t0) / (t1 - t0)) * 100,
+ label: `${String(d.getUTCHours()).padStart(2, '0')}:00`,
+ line: false,
+ })
+ }
+
+ return { bars, skyline: skyline.trim(), max, majors, minors, xticks, unit: '5min', series: [] }
}
/** X ticks for year/all: Monday boundaries up to a quarter, UTC month
diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js
index faabba9..210d5e9 100644
--- a/frontend/src/analytics/format.js
+++ b/frontend/src/analytics/format.js
@@ -25,11 +25,44 @@ export const hostIP = (ip) => {
}
}
-/** Copy the full IP to the clipboard, ignoring failures. */
-export async function copyIp(ip) {
+function showCopiedFeedback(el) {
+ if (!el || typeof document === 'undefined') return
+ const popup = document.createElement('span')
+ popup.textContent = 'Copied!'
+ popup.className = 'copy-popup'
+ popup.style.cssText =
+ 'position:absolute;bottom:calc(100% + 0.25rem);left:50%;' +
+ 'transform:translateX(-50%);padding:0.15rem 0.4rem;' +
+ 'background:var(--text, CanvasText);color:var(--bg, Canvas);' +
+ 'border-radius:0.25rem;font-size:0.75rem;white-space:nowrap;' +
+ 'pointer-events:none;z-index:10;'
+ el.classList.add('has-copy-popup')
+ el.appendChild(popup)
+ setTimeout(() => {
+ popup.remove()
+ el.classList.remove('has-copy-popup')
+ }, 1200)
+}
+
+/** Copy the full IP to the clipboard and show a brief "Copied!" popup. */
+export async function copyIp(ip, event) {
if (!ip) return
+ const el = event?.currentTarget
try {
await navigator.clipboard.writeText(ip)
+ showCopiedFeedback(el)
+ } catch {
+ /* ignore */
+ }
+}
+
+/** Copy arbitrary text to the clipboard and show a brief "Copied!" popup. */
+export async function copyList(text, event) {
+ if (!text) return
+ const el = event?.currentTarget
+ try {
+ await navigator.clipboard.writeText(text)
+ showCopiedFeedback(el)
} catch {
/* ignore */
}
@@ -140,6 +173,33 @@ export function formatWhenTooltip(ts) {
return new Date(ts).toISOString().replace('T', ' ').replace('Z', ' UTC')
}
+/** Full local timestamp for tooltips, e.g. "21 Aug 2026, 17:38:48". */
+export function formatWhenLocal(ts) {
+ return new Date(ts).toLocaleString('en-ie', {
+ year: 'numeric',
+ month: 'short',
+ day: 'numeric',
+ hour: '2-digit',
+ minute: '2-digit',
+ second: '2-digit',
+ })
+}
+
+/** Preserve locale case with the region/country subtag upper-cased. */
+export function formatLang(value) {
+ if (!value || value === '—') return value
+ const parts = value.split('-')
+ if (parts.length > 1) {
+ parts[parts.length - 1] = parts[parts.length - 1].toUpperCase()
+ }
+ return parts.join('-')
+}
+
+/** ISO 8601 UTC timestamp without subseconds, e.g. "2026-08-21T00:20:48Z". */
+export function formatWhenIso(ts) {
+ return `${new Date(ts).toISOString().split('.')[0]}Z`
+}
+
/**
* Format recent visits for display, newest first. Each step is a linked slug
* pointing to its article; external referers/origins are shown as their
@@ -241,8 +301,9 @@ export function formatCrawlerRows(crawlers, pageTree, now = Date.now()) {
.sort((a, b) => totalHits(b) - totalHits(a) || b.lastStart - a.lastStart)
.slice(0, 10)
.map((g) => ({
- when: formatWhen(g.lastStart, now),
- whenTooltip: formatWhenTooltip(g.lastStart),
+ lastSeen: formatWhen(g.lastStart, now),
+ lastSeenIso: formatWhenIso(g.lastStart),
+ lastSeenLocal: formatWhenLocal(g.lastStart),
pages: [...g.pages.entries()]
.sort((a, b) => b[1] - a[1])
.map(([path, count]) => ({
@@ -259,6 +320,85 @@ export function formatCrawlerRows(crawlers, pageTree, now = Date.now()) {
}))
}
+/**
+ * Group abuse hits by IP (never by UA — scanners randomize theirs to
+ * masquerade as legitimate crawlers) and format each group as a row with
+ * the full paths probed, in access order. Flagged paths (the ones that
+ * triggered abuse classification) are lifted to the top, followed by
+ * other 404s, then document GETs from the abuser. UAs are shown raw,
+ * one per line, with their occurrence counts. Paths are shown verbatim
+ * (query string included), not resolved against the page tree.
+ */
+export function formatAbuseRows(abuse, now = Date.now()) {
+ const groups = new Map()
+ for (const a of abuse || []) {
+ const g = groups.get(a.ip) || {
+ ip: a.ip || '',
+ pathHits: [],
+ rawUas: [],
+ uaCounts: new Map(),
+ lastStart: 0,
+ }
+ const start = new Date(a.start).getTime()
+ if (start > g.lastStart) g.lastStart = start
+ g.pathHits.push({
+ path: a.path || '',
+ start,
+ flag: a.flag || false,
+ is_404: a.is_404 || false,
+ })
+ const ua = a.ua || '(no UA)'
+ g.rawUas.push(ua)
+ g.uaCounts.set(ua, (g.uaCounts.get(ua) || 0) + 1)
+ groups.set(a.ip, g)
+ }
+ const totalHits = (g) => g.pathHits.length
+ return [...groups.values()]
+ .sort((a, b) => totalHits(b) - totalHits(a) || b.lastStart - a.lastStart)
+ .slice(0, 10)
+ .map((g) => {
+ const pathCategory = (p) => (p.flag ? 0 : p.is_404 ? 1 : 2)
+ const paths = [...g.pathHits].sort(
+ (a, b) => pathCategory(a) - pathCategory(b) || a.start - b.start,
+ )
+ const uas = [...g.uaCounts.entries()]
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+ return {
+ lastSeen: formatWhen(g.lastStart, now),
+ lastSeenIso: formatWhenIso(g.lastStart),
+ lastSeenLocal: formatWhenLocal(g.lastStart),
+ paths: paths.map((p) => ({ path: p.path, flag: p.flag, is_404: p.is_404 })),
+ allPaths: paths.map((p) => p.path).join('\n'),
+ uas: uas.map(([ua, count]) => ({ ua, count })),
+ allUas: uas
+ .map(([ua, count]) => (count > 1 ? `${count}× ${ua}` : ua))
+ .join('\n'),
+ ip: g.ip,
+ ipDisplay: hostIP(g.ip) || g.ip || '—',
+ total: totalHits(g),
+ }
+ })
+}
+
+/**
+ * Reduce a reverse-DNS hostname to its right-most components that fit
+ * within ``limit`` characters. This keeps the meaningful main domain
+ * while avoiding absurdly long subdomains like ``xxx.yyy.zzz...provider.net``.
+ */
+function mainDomain(host, limit = 24) {
+ if (!host) return host
+ const labels = host.split('.').filter(Boolean)
+ if (!labels.length) return host
+ const parts = [labels.pop()]
+ while (labels.length) {
+ const next = labels[labels.length - 1]
+ const candidate = `${next}.${parts.join('.')}`
+ if (candidate.length > limit) break
+ parts.unshift(labels.pop())
+ }
+ return parts.join('.')
+}
+
/**
* Format raw visit records as rows for a technical table. Returns objects
* with display strings; missing values become "—". ``trail`` starts with the
@@ -268,21 +408,26 @@ export function formatCrawlerRows(crawlers, pageTree, now = Date.now()) {
export function formatVisitRows(visits, pageTree, now = Date.now()) {
const titles = buildTitleMap(pageTree)
return [...(visits || [])].reverse().slice(0, 20).map((v) => {
- const trail = [v.referer, v.entry, ...(v.trail || [])]
+ const trail = [v.entry, ...(v.trail || [])]
.map((p) => stepOf(p, titles))
.filter(Boolean)
const utm = Object.entries(v.utm || {})
.map(([k, value]) => `${k}=${value}`)
.join(', ')
const dash = (s) => (s || '—')
+ const host = v.host || ''
+ const isHost = !!host
return {
- when: formatWhen(v.start, now),
- whenTooltip: formatWhenTooltip(v.start),
+ lastSeen: formatWhen(v.start, now),
+ lastSeenIso: formatWhenIso(v.start),
+ lastSeenLocal: formatWhenLocal(v.start),
+ langDisplay: formatLang(v.lang),
trail,
+ refererStep: stepOf(v.referer, titles),
referer: dash(v.referer),
ip: v.ip || '',
- ipDisplay: v.host || hostIP(v.ip) || v.ip || '—',
- host: dash(v.host),
+ ipDisplay: isHost ? mainDomain(host) : hostIP(v.ip) || v.ip || '—',
+ isHost,
lang: dash(v.lang),
country: dash(v.country),
city: dash(v.city),
diff --git a/frontend/src/analytics/time.js b/frontend/src/analytics/time.js
index 5059dec..1fd80e6 100644
--- a/frontend/src/analytics/time.js
+++ b/frontend/src/analytics/time.js
@@ -13,6 +13,7 @@ export const DAY = 86400e3
export const WEEK = 7 * DAY
export const RANGES = {
+ day: { label: 'day', span: DAY, bucket: MIN5 },
week: { label: 'week' },
month: { label: 'month', span: 30 * DAY, bucket: 6 * HOUR },
year: { label: 'year', span: 365 * DAY, bucket: DAY },
@@ -117,11 +118,36 @@ export function rollingSeries(buckets, rangeKey) {
}
}
-/** Dispatch to weekly or rolling series based on the selected range. */
+/**
+ * Day view: raw 5-minute bucket counts for the current 24-hour window.
+ * No smoothing or rate conversion is applied; counts are used as-is.
+ */
+export function daySeries(buckets) {
+ const raw = rawTimes(buckets)
+ const now = Date.now()
+ const { span, bucket } = RANGES.day
+ const t1 = Math.floor(now / bucket) * bucket + bucket
+ const t0 = t1 - span
+ const points = []
+ for (let t = t0; t < t1; t += bucket) {
+ points.push({ t, count: raw[t] || 0 })
+ }
+ return {
+ series: [{ points, label: '', opacity: 1, area: false }],
+ t0,
+ t1,
+ rate: 1,
+ binMinutes: bucket / 60e3,
+ unitMinutes: bucket / 60e3,
+ unit: '5min',
+ }
+}
+
+/** Dispatch to daily, weekly or rolling series based on the selected range. */
export function makeSeries(buckets, rangeKey) {
- return rangeKey === 'week'
- ? weeklySeries(buckets)
- : rollingSeries(buckets, rangeKey)
+ if (rangeKey === 'day') return daySeries(buckets)
+ if (rangeKey === 'week') return weeklySeries(buckets)
+ return rollingSeries(buckets, rangeKey)
}
/**
diff --git a/frontend/src/analytics/transitions.js b/frontend/src/analytics/transitions.js
index 36397c8..51f94ab 100644
--- a/frontend/src/analytics/transitions.js
+++ b/frontend/src/analytics/transitions.js
@@ -23,22 +23,23 @@ export const TNODE_R = 34 // node circles hold the slug and the view count
export const EXT_R = 34 // external referer/exit nodes use the same full size
// Edge width (half-width of the thin middle) grows logarithmically with
-// the count, anchored so a single recorded transition renders as a ~1 px
-// line. There is no cap — growth is slow enough that even very hot
-// connections stay reasonable. Connections carrying less than
-// PRUNE_FRACTION of the total traffic are not drawn at all (this also
-// keeps the number of drawn connections under ~100).
-const WMID_MIN = 0.5
-const WIDTH_GROWTH = 1.5
+// the count. The constants are scaled down by ~10× so busy ranges (day,
+// year) do not overwhelm the graph with fat connectors. A single recorded
+// transition still renders as a faint ~0.4 px line. Connections carrying
+// less than PRUNE_FRACTION of the total traffic are not drawn at all (this
+// also keeps the graph under ~100 connections).
+const WMID_MIN = 0.2
+const WIDTH_GROWTH = 0.15
const PRUNE_FRACTION = 0.01
// Beads: each edge direction emits beads at count * BEAD_RATE beads per
-// second (linear in the count). The component simulates every bead
-// independently in JS at BEAD_SPEED along the edge, with no limit on
+// second (linear in the count). The rate is reduced ~10× across all time
+// scales to keep the animation lightweight. The component simulates every
+// bead independently in JS at BEAD_SPEED along the edge, with no limit on
// beads in flight.
export const BEAD_SPEED = 180 // svg units per second
export const BEAD_R = 2.2
-const BEAD_RATE = 0.12 // beads per second per recorded transition
+const BEAD_RATE = 0.012 // beads per second per recorded transition
const FLOW_OFFSET = 3 // lane offset to the right of the travel direction
const MAX_EXT_IN = 8 // referer nodes in the top row
@@ -84,10 +85,15 @@ function collectInternalTransitions(transitions) {
return internal
}
-/** Short display label for an external origin (protocol stripped). */
+/** Domain-only label for an external origin (path removed). */
function extLabel(ext) {
- const s = ext.replace(/^https?:\/\//, '')
- return s.length > 11 ? `${s.slice(0, 10)}…` : s
+ try {
+ const host = new URL(ext).hostname
+ return host.length > 25 ? `${host.slice(0, 24)}…` : host
+ } catch {
+ const s = ext.replace(/^https?:\/\//, '').split('/')[0]
+ return s.length > 25 ? `${s.slice(0, 24)}…` : s
+ }
}
/**
@@ -175,8 +181,27 @@ function layoutAngles(root, unit, weight) {
}
}
+/** Compute median reading time per article in whole minutes. */
+function buildReadMinutes(visits) {
+ const times = {}
+ for (const v of visits || []) {
+ for (const [path, sec] of Object.entries(v.read || {})) {
+ ;(times[path] || (times[path] = [])).push(sec)
+ }
+ }
+ const minutes = {}
+ for (const [path, arr] of Object.entries(times)) {
+ arr.sort((a, b) => a - b)
+ const mid = Math.floor(arr.length / 2)
+ const median =
+ arr.length % 2 ? arr[mid] : (arr[mid - 1] + arr[mid]) / 2
+ minutes[path] = Math.max(1, Math.round(median / 60))
+ }
+ return minutes
+}
+
/** Compute radial positions, view counts and labels for each node. */
-function positionNodes(nodes, maxDepth, unit, viewsData, titles) {
+function positionNodes(nodes, maxDepth, unit, viewsData, titles, readMinutes) {
// Constant radial gap between rings, equal to the arc spacing of nodes
// along a ring: leaf arc = unit * GAP, so GAP scales up with `unit` on
// sparse trees (where closing the circle forces wider arcs) and with
@@ -196,10 +221,14 @@ function positionNodes(nodes, maxDepth, unit, viewsData, titles) {
n.x = Math.cos(n.angle) * r
n.y = Math.sin(n.angle) * r
n.views = viewCount(n.path)
+ n.readMin = readMinutes[n.path] || 0
// Slug inside the circle; full title goes on the link title attribute.
const slug = n.path === '/' ? '🏠' : n.path.split('/').pop()
- n.label = slug.length > 11 ? `${slug.slice(0, 10)}…` : slug
+ n.label = slug.length > 16 ? `${slug.slice(0, 15)}…` : slug
n.title = titles.get(n.path) || ''
+ // Category (non-leaf) pages with no views in this window are left
+ // blank to keep the layout, but their circle/label is not drawn.
+ n.hidden = n.children.length > 0 && n.views === 0
}
return { radius, GAP }
@@ -279,7 +308,7 @@ const fmtPt = (p) => `${p[0].toFixed(2)} ${p[1].toFixed(2)}`
* `wMid` is the half-width of the thin middle (already strength-scaled by
* the caller); `ra`/`rb` are the radii of the node circles each end wraps.
*/
-function buildRibbon(a, b, ab, ba, wMid, ra = TNODE_R, rb = TNODE_R) {
+function buildRibbon(a, b, ab, ba, wMid, ra = TNODE_R, rb = TNODE_R, external = false) {
const count = ab + ba
const len = Math.hypot(b.x - a.x, b.y - a.y) || 1
const ux = (b.x - a.x) / len
@@ -387,6 +416,7 @@ function buildRibbon(a, b, ab, ba, wMid, ra = TNODE_R, rb = TNODE_R) {
return {
d,
title: `${a.path} ↔ ${b.path}: ${count} (${ab} / ${ba})`,
+ external,
}
}
@@ -401,7 +431,7 @@ function buildRibbon(a, b, ab, ba, wMid, ra = TNODE_R, rb = TNODE_R) {
* edge run on parallel lanes instead of colliding. The component turns
* these into independently simulated beads.
*/
-function buildFlows(a, b, ra, rb, ab, ba) {
+function buildFlows(a, b, ra, rb, ab, ba, visualScale = 1) {
const len = Math.hypot(b.x - a.x, b.y - a.y) || 1
const ux = (b.x - a.x) / len
const uy = (b.y - a.y) / len
@@ -422,7 +452,7 @@ function buildFlows(a, b, ra, rb, ab, ba) {
x2: a.x + toT * ux + s * rx,
y2: a.y + toT * uy + s * ry,
len: span,
- interval: 1 / (count * BEAD_RATE),
+ interval: 1 / (count * BEAD_RATE * visualScale),
}
}
const flows = []
@@ -437,14 +467,17 @@ function buildFlows(a, b, ra, rb, ab, ba) {
* Absolute on purpose — cool routes stay visible regardless of how hot
* the hottest connection is.
*/
-const scaledWidth = (count) => WMID_MIN + WIDTH_GROWTH * Math.log1p(count - 1)
+const scaledWidth = (count) => {
+ if (count <= 0) return 0
+ return WMID_MIN + WIDTH_GROWTH * Math.log1p(count - 1)
+}
/**
* Build ribbon edges and bead flows for every aggregated page-to-page
* pair. Pairs carrying less than PRUNE_FRACTION of the total internal
* traffic are pruned (this naturally bounds the graph to ~100 edges).
*/
-function buildInternalEdges(pairs, byPath) {
+function buildInternalEdges(pairs, byPath, visualScale = 1) {
let total = 0
for (const [, [ab, ba]] of pairs) total += ab + ba
const minCount = total * PRUNE_FRACTION
@@ -456,8 +489,10 @@ function buildInternalEdges(pairs, byPath) {
const [pf, pt] = k.split(' ')
const a = byPath.get(pf)
const b = byPath.get(pt)
- edges.push(buildRibbon(a, b, ab, ba, scaledWidth(ab + ba)))
- flows.push(...buildFlows(a, b, TNODE_R, TNODE_R, ab, ba))
+ const wMid = scaledWidth((ab + ba) * visualScale)
+ if (wMid <= 0) continue
+ edges.push(buildRibbon(a, b, ab, ba, wMid))
+ flows.push(...buildFlows(a, b, TNODE_R, TNODE_R, ab, ba, visualScale))
}
return { edges, flows }
}
@@ -507,7 +542,7 @@ export function filterViewsByRange(views, t0, t1) {
* Widths and pruning use the same log scale and traffic-share rule as
* internal connections.
*/
-function buildExternal(external, byPath, radius, innerBounds) {
+function buildExternal(external, byPath, radius, innerBounds, visualScale = 1) {
const extNodes = []
const edges = []
const flows = []
@@ -517,7 +552,7 @@ function buildExternal(external, byPath, radius, innerBounds) {
const live = external.filter((p) => byPath.has(p.page))
if (!live.length) return { extNodes, edges, flows }
- const width = scaledWidth
+ const width = (count) => scaledWidth(count * visualScale)
const overlaps = (x, y, r) =>
[...byPath.values(), ...extNodes].some(
@@ -547,8 +582,10 @@ function buildExternal(external, byPath, radius, innerBounds) {
extNodes.push(xn)
for (const p of ps) {
const page = byPath.get(p.page)
- edges.push(buildRibbon(xn, page, p.in, 0, width(p.in), EXT_R, TNODE_R))
- flows.push(...buildFlows(xn, page, EXT_R, TNODE_R, p.in, 0))
+ const wMid = width(p.in)
+ if (wMid <= 0) continue
+ edges.push(buildRibbon(xn, page, p.in, 0, wMid, EXT_R, TNODE_R, true))
+ flows.push(...buildFlows(xn, page, EXT_R, TNODE_R, p.in, 0, visualScale))
}
})
}
@@ -593,8 +630,10 @@ function buildExternal(external, byPath, radius, innerBounds) {
extNodes.push(xn)
}
xn.count += p.out
- edges.push(buildRibbon(page, xn, p.out, 0, width(p.out), TNODE_R, EXT_R))
- flows.push(...buildFlows(page, xn, TNODE_R, EXT_R, p.out, 0))
+ const wMid = width(p.out)
+ if (wMid <= 0) continue
+ edges.push(buildRibbon(page, xn, p.out, 0, wMid, TNODE_R, EXT_R, true))
+ flows.push(...buildFlows(page, xn, TNODE_R, EXT_R, p.out, 0, visualScale))
}
return { extNodes, edges, flows }
@@ -605,11 +644,12 @@ function buildExternal(external, byPath, radius, innerBounds) {
* Returns { nodes, edges, flows, extNodes, arcs, bounds } or null when
* there is nothing to show.
*/
-export function buildTransitionGraph(data, pageTree) {
+export function buildTransitionGraph(data, pageTree, visits = [], visualScale = 1) {
const internal = collectInternalTransitions(data?.transitions)
const external = collectExternalPairs(data?.transitions)
const navOrder = buildNavigationOrder(pageTree)
const titles = buildTitleMap(pageTree)
+ const readMinutes = buildReadMinutes(visits)
if (!internal.length && !navOrder.size) return null
@@ -619,10 +659,10 @@ export function buildTransitionGraph(data, pageTree) {
layoutAngles(root, unit, weightFn)
const maxDepth = Math.max(1, ...nodes.map((n) => n.depth))
- const { radius } = positionNodes(nodes, maxDepth, unit, data?.views, titles)
+ const { radius } = positionNodes(nodes, maxDepth, unit, data?.views, titles, readMinutes)
const arcs = buildFamilyArcs(nodes, radius)
const pairs = aggregatePairs(internal)
- const { edges, flows } = buildInternalEdges(pairs, byPath)
+ const { edges, flows } = buildInternalEdges(pairs, byPath, visualScale)
// Tight bounding box of the actual page nodes; family ring arcs can sweep
// outside the node circle (e.g. a large arc between two siblings on the
@@ -646,7 +686,7 @@ export function buildTransitionGraph(data, pageTree) {
bounds.y1 = Math.max(bounds.y1, b.y1)
}
- const ext = buildExternal(external, byPath, radius, bounds)
+ const ext = buildExternal(external, byPath, radius, bounds, visualScale)
for (const xn of ext.extNodes) {
bounds.x0 = Math.min(bounds.x0, xn.x - xn.r - pad)
bounds.y0 = Math.min(bounds.y0, xn.y - xn.r - pad)
diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js
index c2645bc..0bfebf4 100644
--- a/frontend/src/pagerite.js
+++ b/frontend/src/pagerite.js
@@ -338,29 +338,107 @@ import "overlayscrollbars/overlayscrollbars.css";
}
// --- Analytics pings ---------------------------------------------------
- // Fire-and-forget POST /_a {fr, to}: on the initial page load (starts the
- // visit — the server counts nothing from the document GET alone), for
- // internal fetch-navigations and for external https exits. Excluded:
- // back/forward (popstate never pings) and everything while we know the
- // user is an admin — but only when SSO is actually in use; with no auth
- // (dev/test) "admin" is everyone's state and nothing would be recorded —
- // or has the editor open (admin noise, not visits). The analytics page
- // itself (/_a) is also excluded even though fetch-navigation treats it like
- // a normal article.
+ // Fire-and-forget POST /_a {fr, to, read}: on the initial page load
+ // (starts the visit — the server counts nothing from the document GET
+ // alone), for internal fetch-navigations, for external https exits, and
+ // on window close. ``read`` is the active time (ms) spent on ``fr``.
+ // Reading time pauses after 1 minute of inactivity and resumes on the
+ // next mouse/touch/scroll/keyboard event.
+ // Excluded: back/forward (popstate never pings), everything while the
+ // editor is open (body.editing — admin noise, not visits), and the
+ // analytics page itself (/_a), even though fetch-navigation treats it
+ // like a normal article.
+ // Admins (when SSO is actually in use — with no auth proxy "admin" is
+ // everyone's state) ping normally but with hide=1: the server then
+ // records nothing and scrubs any session the same browser accumulated
+ // before logging in, so admins never show up as visits or crawlers.
// See docs/analytics.md.
- function ping(to, fr = currentPath) {
- if ((ssoAvailable && isAdmin) || document.body.classList.contains("editing")
- || to === "/_a" || fr === "/_a") return;
+ function ping(to, fr = currentPath, read = 0) {
+ if (document.body.classList.contains("editing")) return;
+ if ((to && to === "/_a") || fr === "/_a") return;
+ const hide = ssoAvailable && isAdmin ? 1 : 0;
+ const body = JSON.stringify({
+ fr, to, hide,
+ read: Math.max(0, Math.round(read / 1000)),
+ });
try {
fetch("/_a", {
method: "POST",
keepalive: true,
headers: { "content-type": "application/json" },
- body: JSON.stringify({ fr, to }),
+ body,
});
} catch { /* analytics must never break navigation */ }
}
+ // Active reading time for the current page. The clock stops after 1 minute
+ // without activity and restarts on the next mouse/touch/scroll/keyboard
+ // event.
+ const INACTIVE_MS = 60_000;
+ let readStart = performance.now();
+ let readElapsed = 0;
+ let reading = true;
+ let readInactivityTimer = null;
+ let closePingedFor = null;
+
+ function markReadActivity() {
+ if (!reading) {
+ reading = true;
+ readStart = performance.now();
+ }
+ clearTimeout(readInactivityTimer);
+ readInactivityTimer = setTimeout(() => {
+ if (reading) {
+ readElapsed += performance.now() - readStart;
+ reading = false;
+ }
+ }, INACTIVE_MS);
+ }
+
+ function takeReadTime() {
+ if (reading) {
+ readElapsed += performance.now() - readStart;
+ readStart = performance.now();
+ }
+ const ms = Math.max(0, Math.round(readElapsed));
+ readElapsed = 0;
+ return ms;
+ }
+
+ function resetReadTime() {
+ readElapsed = 0;
+ reading = true;
+ readStart = performance.now();
+ clearTimeout(readInactivityTimer);
+ }
+
+ function sendClosePing() {
+ if (closePingedFor === currentPath) return;
+ const read = Math.max(0, Math.round(takeReadTime() / 1000));
+ if (read <= 0) return;
+ const hide = ssoAvailable && isAdmin ? 1 : 0;
+ const body = JSON.stringify({ fr: currentPath, hide, read });
+ const blob = new Blob([body], { type: "application/json" });
+ try {
+ if (navigator.sendBeacon) {
+ navigator.sendBeacon("/_a", blob);
+ } else {
+ fetch("/_a", {
+ method: "POST",
+ keepalive: true,
+ headers: { "content-type": "application/json" },
+ body,
+ });
+ }
+ } catch { /* analytics must never break navigation */ }
+ closePingedFor = currentPath;
+ }
+
+ for (const ev of ["mousemove", "mousedown", "touchstart", "touchmove", "scroll", "keydown"]) {
+ addEventListener(ev, markReadActivity, { passive: true });
+ }
+ addEventListener("pagehide", sendClosePing);
+
// The initial page load pings too — it is what starts the visit and
// counts the entry page view (the document GET alone records nothing).
// Sent once per load, after the auth probes so the admin gate applies;
@@ -538,7 +616,10 @@ import "overlayscrollbars/overlayscrollbars.css";
if (url.origin !== location.origin) {
// External link: the browser navigates; record the full https URL so
// different links to the same domain stay distinct in analytics.
- if (url.protocol === "https:") ping(url.href);
+ if (url.protocol === "https:") {
+ closePingedFor = currentPath;
+ ping(url.href, currentPath, takeReadTime());
+ }
return;
}
// Same-page anchor links (footnotes etc.): let the browser handle them
@@ -550,7 +631,12 @@ import "overlayscrollbars/overlayscrollbars.css";
ev.preventDefault();
// Capture the source now: load() updates currentPath before pinging.
const from = currentPath;
- load(url).then((ok) => { if (ok) ping(url.pathname, from); });
+ load(url).then((ok) => {
+ if (!ok) return;
+ closePingedFor = null;
+ ping(url.pathname, from, takeReadTime());
+ resetReadTime();
+ });
});
addEventListener("popstate", () => {
diff --git a/pagerite/analytics.py b/pagerite/analytics.py
index 2ce8bd0..5daf767 100644
--- a/pagerite/analytics.py
+++ b/pagerite/analytics.py
@@ -5,10 +5,14 @@ ping on page load starts a visit, later pings extend it, and pings with no
known session start a fresh one (missing data, not dropped). The document
GET handler stashes the entry referer (external https origin) and any
utm_* query parameters in in-memory IP tables, consumed when the ping
-starts the visit; nothing is counted without a ping (bots and admin
-browsing stay invisible). The session map is in-memory only. The visitor
-IP and, when available, its reverse-DNS host name are stored on the visit
-record itself.
+starts the visit; nothing is counted without a ping (bots stay invisible).
+Admin clients ping with ``hide=1``, which records nothing and removes any
+visit the session accumulated before logging in. Scanner telltale 404s
+(dotpaths, *.php) classify the source IP as abuse; its hits — including
+earlier crawler hits — are moved to the abuse list, which the viewer
+groups by IP with full request paths. The session map is in-memory only.
+The visitor IP and, when available, its reverse-DNS host name are stored
+on the visit record itself.
Data is a msgspec Struct JSON-dumped to its own file (not the kanta db),
rewritten atomically on every recorded event.
@@ -41,7 +45,10 @@ def _compact_user_agent(ua: str) -> str:
dev = r.device.family if r.device else None
if browser in (None, "Other") and os_name in (None, "Other"):
return ua
- browser = browser if browser and browser != "Other" else ""
+ if browser and browser != "Other":
+ browser = browser.split()[0]
+ else:
+ browser = ""
os_name = os_name if os_name and os_name != "Other" else ""
if dev in (None, "Other") or dev == browser:
dev = ""
@@ -79,6 +86,8 @@ class Visit(msgspec.Struct, omit_defaults=True):
ua_pretty: str = ""
#: UTM query parameters from the landing URL, keyed by parameter name.
utm: dict[str, str] = {}
+ #: Active reading time per path (seconds), keyed by path.
+ read: dict[str, int] = {}
class CrawlerHit(msgspec.Struct, omit_defaults=True):
@@ -96,6 +105,29 @@ class CrawlerHit(msgspec.Struct, omit_defaults=True):
query: str = ""
+class AbuseHit(msgspec.Struct, omit_defaults=True):
+ """A request from an IP classified as a scanner/abuser.
+
+ Unlike crawler hits the full request path (query string included) is
+ kept: the interesting part is exactly which paths were probed.
+ ``flag`` marks the path that triggered classification; ``is_404``
+ distinguishes 404 responses from document GETs made by the abuser.
+ """
+
+ start: datetime
+ #: Full request path including the query string (e.g. "/.env?x=1").
+ path: str
+ ip: str = ""
+ ua: str = ""
+ #: Compact display form of ``ua`` when parsable.
+ ua_pretty: str = ""
+ #: True when this path triggered abuse classification (telltale path
+ #: or the 404 that crossed the threshold).
+ flag: bool = False
+ #: True for 404 responses; false for document GETs from the abuser.
+ is_404: bool = False
+
+
class Analytics(msgspec.Struct, omit_defaults=True):
"""Root of the analytics JSON file. Append-only by design: old data is
dropped by deleting list entries / bucket keys."""
@@ -103,6 +135,10 @@ class Analytics(msgspec.Struct, omit_defaults=True):
visits: list[Visit] = []
#: Document GETs that never produced a ping, treated as crawler/bot hits.
crawlers: list[CrawlerHit] = []
+ #: Requests from abusive IPs (see AbuseHit), grouped by IP in the viewer.
+ abuse: list[AbuseHit] = []
+ #: IPs classified as scanners/abusers (keys; values always True).
+ abuse_ips: dict[str, bool] = {}
#: Page transitions per 5-minute bucket (sparse):
#: from -> to -> bucket ISO -> count. ``from`` is the referer origin or
#: "(direct)" for initial loads, a page path for pings.
@@ -186,6 +222,19 @@ def _utm_tags(query: str) -> dict[str, str]:
_CRAWLER_TIMEOUT = timedelta(seconds=10)
+#: Plain-404 count per IP that classifies it as abuse even without a
+#: telltale path hit.
+_ABUSE_404_THRESHOLD = 10
+
+#: Paths that instantly classify an IP as abuse when they 404: any segment
+#: starting with a dot ("/.env", "/.git/config") or ending in ".php".
+_ABUSE_PATH = re.compile(r"(^|/)\.|\.php$", re.IGNORECASE)
+
+
+def _is_abuse_path(path: str) -> bool:
+ """Telltale scanner path: dot segment or *.php."""
+ return bool(_ABUSE_PATH.search(path.split("?")[0]))
+
class Store:
"""In-memory analytics data plus the (IP, UA) -> visit session map."""
@@ -212,6 +261,9 @@ class Store:
#: Document GETs that have not yet been matched by a ping. Kept
#: in RAM only; expired entries are written to ``data.crawlers``.
self.pending_crawlers: list[CrawlerHit] = []
+ #: ip -> number of plain (non-telltale) 404s seen, in RAM only;
+ #: reaching ``_ABUSE_404_THRESHOLD`` classifies the IP as abuse.
+ self.not_found_counts: dict[str, int] = {}
#: Callables to notify when persisted data changes. Registered by the
#: analytics WebSocket broadcaster.
self._on_change: list[Callable[[], None]] = []
@@ -267,6 +319,123 @@ class Store:
buckets = self.data.transitions.setdefault(fr, {}).setdefault(to, {})
self._count(buckets, _bucket(now))
+ def _uncount(self, table: dict[str, int], key: str) -> None:
+ """Reverse one ``_count``: decrement and drop empty keys."""
+ if key in table:
+ table[key] -= 1
+ if table[key] <= 0:
+ del table[key]
+
+ def _remove_visit(self, index: int) -> None:
+ """Delete a visit and reverse the counts its creation recorded.
+
+ Used when a known visitor turns out to be an admin (hide=1 ping):
+ the session is scrubbed from the stats. Views/transitions logged
+ by later pings inside the visit lack per-event timestamps and are
+ left as-is.
+ """
+ visit = self.data.visits[index]
+ bucket = _bucket(visit.start)
+ self._uncount(self.data.site_visits, bucket)
+ views = self.data.views.get(visit.entry)
+ if views is not None:
+ self._uncount(views, bucket)
+ if not views:
+ del self.data.views[visit.entry]
+ fr_map = self.data.transitions.get(visit.referer or "(direct)")
+ if fr_map is not None:
+ buckets = fr_map.get(visit.entry)
+ if buckets is not None:
+ self._uncount(buckets, bucket)
+ if not buckets:
+ del fr_map[visit.entry]
+ if not fr_map:
+ del self.data.transitions[visit.referer or "(direct)"]
+ del self.data.visits[index]
+ # Sessions store list indices; shift the ones past the removed visit.
+ for key, i in list(self.sessions.items()):
+ if i > index:
+ self.sessions[key] = i - 1
+
+ def _abuse_hit(
+ self,
+ ip: str,
+ ua: str,
+ path: str,
+ start: datetime | None = None,
+ *,
+ flag: bool = False,
+ is_404: bool = False,
+ ) -> None:
+ """Append one abuse hit with the full request path."""
+ self.data.abuse.append(
+ AbuseHit(
+ start=start or datetime.now(UTC),
+ path=path,
+ ip=ip,
+ ua=ua,
+ ua_pretty=_compact_user_agent(ua),
+ flag=flag,
+ is_404=is_404,
+ )
+ )
+
+ def classify_abuse(
+ self,
+ ip: str,
+ ua: str,
+ path: str,
+ *,
+ flag: bool = False,
+ is_404: bool = False,
+ ) -> None:
+ """Classify an IP as a scanner/abuser and record the triggering hit.
+
+ All earlier crawler hits from the same IP (persisted and pending)
+ are moved to the abuse list — a random-UA scanner must not pollute
+ the crawler stats of the legitimate bots it impersonates.
+ """
+ if ip not in self.data.abuse_ips:
+ self.data.abuse_ips[ip] = True
+ moved = [h for h in self.data.crawlers if h.ip == ip]
+ if moved:
+ self.data.crawlers = [h for h in self.data.crawlers if h.ip != ip]
+ for h in moved:
+ self._abuse_hit(
+ h.ip, h.ua,
+ h.entry + (f"?{h.query}" if h.query else ""),
+ start=h.start,
+ )
+ pending = [h for h in self.pending_crawlers if h.ip == ip]
+ if pending:
+ self.pending_crawlers = [h for h in self.pending_crawlers if h.ip != ip]
+ for h in pending:
+ self._abuse_hit(
+ h.ip, h.ua,
+ h.entry + (f"?{h.query}" if h.query else ""),
+ start=h.start,
+ )
+ self._abuse_hit(ip, ua, path, flag=flag, is_404=is_404)
+ self._save()
+
+ def track_404(self, ip: str, ua: str, path: str) -> None:
+ """Record a 404 response for ``path`` (full path, query included).
+
+ A telltale path (dot segment or *.php) classifies the IP as abuse
+ immediately; enough plain 404s from one IP do too. Hits from
+ already-classified IPs go straight to the abuse list.
+ """
+ if ip in self.data.abuse_ips:
+ self._abuse_hit(ip, ua, path, flag=_is_abuse_path(path), is_404=True)
+ self._save()
+ return
+ if _is_abuse_path(path):
+ self.classify_abuse(ip, ua, path, flag=True, is_404=True)
+ return
+ self.not_found_counts[ip] = self.not_found_counts.get(ip, 0) + 1
+ if self.not_found_counts[ip] >= _ABUSE_404_THRESHOLD:
+ self.classify_abuse(ip, ua, path, flag=True, is_404=True)
+
def _new_visit(
self,
entry: str,
@@ -328,8 +497,7 @@ class Store:
own_origin: str,
ip: str,
ua: str,
- entry: str,
- query: str = "",
+ full_path: str,
) -> None:
"""Stash the entry referer/UTM tags and queue a pending crawler hit.
@@ -343,7 +511,17 @@ class Store:
Every document GET is also queued as a pending crawler hit. If a ping
from the same (IP, UA) pair arrives within ``_CRAWLER_TIMEOUT``, the
hit is discarded; otherwise it is flushed to ``data.crawlers``.
+
+ GETs from IPs already classified as abuse are recorded as abuse hits
+ with the full request path (query string included).
"""
+ entry = full_path.split("?")[0]
+ query = full_path.split("?", 1)[1] if "?" in full_path else ""
+ if ip in self.data.abuse_ips:
+ self._flush_crawlers()
+ self._abuse_hit(ip, ua, full_path, is_404=False, flag=False)
+ self._save()
+ return
now = datetime.now(UTC)
self._flush_crawlers(now)
if referer:
@@ -365,31 +543,73 @@ class Store:
)
)
+ def _add_read(self, ip: str, ua: str, path: str, seconds: int) -> None:
+ """Add ``seconds`` of reading time for ``path`` to the current visit."""
+ if seconds <= 0:
+ return
+ index = self.sessions.get((ip, ua))
+ if index is None or index >= len(self.data.visits):
+ return
+ visit = self.data.visits[index]
+ visit.read[path] = visit.read.get(path, 0) + seconds
+
def ping(
self,
from_: str,
- to: str,
+ to: str | None,
ip: str,
ua: str,
accept_language: str = "",
+ hide: bool = False,
+ read: int = 0,
) -> int | None:
- """Record a client navigation ping ({from, to} from pagerite.js).
+ """Record a client navigation ping ({from, to, read} from pagerite.js).
+
+ ``to`` is an internal path ("/...") or an https URL for exit links; a
+ missing/empty ``to`` means the page is being closed and only the
+ ``read`` time should be recorded. The transition is always counted when
+ ``to`` is present; the trail only grows on first sight of a page within
+ the visit. ``read`` is the active time (seconds) spent on ``from_``.
- ``to`` is an internal path ("/...") or an https URL for exit links;
- anything else is ignored. The transition is always counted; the trail
- only grows on first sight of a page within the visit.
A ping with no known session starts a fresh visit, consuming the
referer and UTM tags stashed by the document GET if there are any.
+ ``hide`` is set by admin clients: the ping cancels pending crawler
+ hits as usual, and any existing visit for this (IP, UA) session is
+ removed from the stats (the admin browsed anonymously before logging
+ in). Nothing new is recorded.
+
+ Pings from IPs classified as abuse are ignored entirely.
+
Returns the index of the new visit when one is created, so callers
can enrich it later with non-blocking lookups (host, geoip country).
"""
self._flush_crawlers()
+ key = (ip, ua)
+ if hide:
+ # Admin ping: cancel pending crawler hits and scrub the session.
+ self.pending_crawlers = [
+ hit for hit in self.pending_crawlers if not (hit.ip == ip and hit.ua == ua)
+ ]
+ index = self.sessions.pop(key, None)
+ if index is not None and index < len(self.data.visits):
+ self._remove_visit(index)
+ self._save()
+ return None
+ if ip in self.data.abuse_ips:
+ return None
# A real visitor ping cancels any pending crawler hits from this
# (IP, UA) pair.
self.pending_crawlers = [
hit for hit in self.pending_crawlers if not (hit.ip == ip and hit.ua == ua)
]
+ fr_path = _internal_path(from_) if from_ else ""
+ if fr_path and read > 0:
+ self._add_read(ip, ua, fr_path, read)
+ if not to:
+ if read > 0:
+ self._save()
+ return None
if to.startswith("/") and not to.startswith("//"):
target = _internal_path(to) or ""
else:
@@ -398,7 +618,7 @@ class Store:
return None
key = (ip, ua)
index = self.sessions.get(key)
- fr = (_internal_path(from_) or "(direct)") if from_ else "(direct)"
+ fr = fr_path or "(direct)"
if index is None or index >= len(self.data.visits):
# No known session: the initial ping of a fresh page load (or
# missing data after a server restart) — start a visit.
diff --git a/pagerite/app.py b/pagerite/app.py
index 36a9db0..6837d87 100644
--- a/pagerite/app.py
+++ b/pagerite/app.py
@@ -126,13 +126,21 @@ class GeoIP:
return ""
def city(self, ip: str) -> str:
- """City name for ``ip``, or "" when unavailable."""
+ """City name for ``ip``, or "" when unavailable.
+
+ GeoIP sometimes appends district names in parentheses (e.g.
+ "Berlin (Bezirk Tempelhof-Schöneberg)"); those are stripped before
+ the value is stored.
+ """
if not ip or self._reader is None:
return ""
try:
rec = self._reader.get(ip)
if rec:
- return (rec.get("city") or {}).get("names", {}).get("en", "")
+ city = (rec.get("city") or {}).get("names", {}).get("en", "")
+ if city:
+ city = re.sub(r"\s*\([^)]*\)", "", city).strip()
+ return city
except Exception:
pass
return ""
@@ -605,6 +613,12 @@ def _client_ip(request: Request) -> str:
return forwarded or (request.client.host if request.client else "")
+def _query_suffix(request: Request) -> str:
+ """The request's query string as a "?..." suffix, or "" when absent."""
+ query = str(request.url.query)
+ return f"?{query}" if query else ""
+
+
@lru_cache(maxsize=4096)
def _cached_ptr(ip: str) -> str:
"""Reverse-DNS lookup with in-RAM LRU cache. Returns the host name or ""."""
@@ -683,7 +697,11 @@ class AnalyticsPing(BaseModel):
"""Navigation ping from pagerite.js (see docs/analytics.md)."""
fr: str = ""
- to: str
+ to: str | None = None
+ #: 1 from admin clients: scrub the session instead of recording it.
+ hide: int = 0
+ #: Active reading time on ``fr`` (ms), if any.
+ read: int = 0
@app.get("/_a", response_model=None)
@@ -716,6 +734,8 @@ async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
ip,
request.headers.get("user-agent", ""),
request.headers.get("accept-language", ""),
+ hide=bool(ping.hide),
+ read=ping.read,
)
if index is not None:
asyncio.create_task(_enrich_visit(index, ip))
@@ -725,7 +745,8 @@ def _track_entry(path: str, request: Request) -> None:
"""Stash the referer/UTM tags and queue a pending crawler hit for the GET.
Nothing is counted on the GET itself — the client's /_a ping starts the
- visit, so bots and admin browsing never register as visits.
+ visit, so bots never register as visits. (Admin clients ping too, but
+ with hide=1, which scrubs their session instead of recording it.)
The devserver's health probe (``GET /?from=devserver.py`` from
``127.0.0.1``) is ignored: it is not real traffic and would otherwise be
@@ -739,13 +760,13 @@ def _track_entry(path: str, request: Request) -> None:
):
return
own_origin = f"https://{urlparse(str(request.base_url)).netloc}"
+ full_path = f"{request.url.path}{_query_suffix(request)}"
analytics_store.track_entry(
request.headers.get("referer", ""),
own_origin,
_client_ip(request),
request.headers.get("user-agent", ""),
- "/" if path == "" else f"/{path}",
- str(request.url.query),
+ full_path,
)
@@ -968,6 +989,13 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
if path and _is_reserved(path):
# Invalid slug shape: not a content URL, let FastAPI return its
# built-in 404 instead of rendering an editable article page.
+ # Scanner telltales (dotpaths like /.env, *.php) classify the IP
+ # as abuse in analytics.
+ analytics_store.track_404(
+ _client_ip(request),
+ request.headers.get("user-agent", ""),
+ f"/{path}{_query_suffix(request)}",
+ )
raise HTTPException(404)
chain = resolve(data.menu, path)
node = chain[-1] if chain else None
@@ -1011,5 +1039,10 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response:
if item.published:
return RedirectResponse(f"/{slug}")
if _is_trackable_path(path):
+ analytics_store.track_404(
+ _client_ip(request),
+ request.headers.get("user-agent", ""),
+ f"/{path}{_query_suffix(request)}",
+ )
_track_entry(path, request)
return HTMLResponse(views.render_not_found(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html), 404)