From b4e8fad09096e0692024f662b8ee8a2197485505 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Thu, 20 Aug 2026 18:13:41 +0000 Subject: [PATCH] Implement analytics feature Add server-side visit analytics collection, a public-page ping endpoint, and a full-screen AnalyticsView for admins. Backend: - Add pagerite/analytics.py: Analytics/Visit model, Store, and persistence - Wire /_a ping endpoint and GET /_api/analytics into pagerite/app.py Frontend: - Add full-screen AnalyticsView with visitor charts and transition map - Add VisitorCharts and TransitionGraph subcomponents - Add analytics JS helpers in frontend/src/analytics/ - Send navigation pings from frontend/src/pagerite.js - Mount AnalyticsView from frontend/src/main.js - Document the feature in docs/analytics.md and update AGENTS.md --- .gitignore | 1 + AGENTS.md | 3 +- docs/analytics.md | 122 ++++++++ frontend/src/AnalyticsView.vue | 222 +++++++++++++++ frontend/src/TransitionGraph.vue | 100 +++++++ frontend/src/VisitorCharts.vue | 189 ++++++++++++ frontend/src/analytics/chart.js | 333 ++++++++++++++++++++++ frontend/src/analytics/format.js | 54 ++++ frontend/src/analytics/time.js | 141 +++++++++ frontend/src/analytics/transitions.js | 394 ++++++++++++++++++++++++++ frontend/src/assets/pagerite.css | 8 +- frontend/src/main.js | 50 ++++ frontend/src/pagerite.js | 93 +++++- frontend/vite.config.js | 5 +- pagerite/analytics.py | 174 ++++++++++++ pagerite/app.py | 60 +++- 16 files changed, 1932 insertions(+), 17 deletions(-) create mode 100644 docs/analytics.md create mode 100644 frontend/src/AnalyticsView.vue create mode 100644 frontend/src/TransitionGraph.vue create mode 100644 frontend/src/VisitorCharts.vue create mode 100644 frontend/src/analytics/chart.js create mode 100644 frontend/src/analytics/format.js create mode 100644 frontend/src/analytics/time.js create mode 100644 frontend/src/analytics/transitions.js create mode 100644 pagerite/analytics.py diff --git a/.gitignore b/.gitignore index 7f56dc8..851b616 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ !.gitignore *.lock *.kantadb +pagerite.analytics.json /pagerite/frontend-build package-lock.json diff --git a/AGENTS.md b/AGENTS.md index cec7766..a3f93c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,8 +15,9 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `markdown.py` — markdown-it-py renderer. - `views.py` — shared page layout and rendering. - `seed.py` — demo content, written only on first database creation. + - `analytics.py` — visit analytics collection (see `docs/analytics.md`). - `frontend/src/` — Vue editor and public-page JS entries. - - `main.js` — Vue editor app entry. + - `main.js` — Vue editor app entry (also mounts the full-screen AnalyticsView). - `pagerite.js` — public page entry. - `assets/` — base CSS, Pygments styles, fonts. - `scripts/devserver.py` — dev server with auto reload (the user mostly uses this; avoid running the server yourself, ask the user to test). diff --git a/docs/analytics.md b/docs/analytics.md new file mode 100644 index 0000000..a0f1a48 --- /dev/null +++ b/docs/analytics.md @@ -0,0 +1,122 @@ +# Analytics + +Server-side visit analytics. Data lives in a plain JSON file — a msgspec +Struct dumped to disk — separate from the kanta content database, path from +`PAGERITE_ANALYTICS` (default: the database path with `.kantadb` replaced by +`.analytics.json`, e.g. `pagerite.analytics.json`). + +- `pagerite/analytics.py` — data model (`Analytics`, `Visit`) and the `Store` + (in-memory data + session map, atomic JSON persistence). +- `pagerite/app.py` — entry-referer stashing in `show_page` (`_track_entry`), + the `POST /_a` ping endpoint, and `GET /_api/analytics` (admin-gated like + every `/_api` endpoint). +- `frontend/src/pagerite.js` — client navigation pings and the 📊 pen. +- `frontend/src/AnalyticsView.vue` — full-screen viewer (its own Vue app via + `openAnalytics()`/`closeAnalytics()` in `main.js`, not a docked-panel tab). + +## What is collected + +The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with +`{fr, to}` (`fr` = source path): + +- **Initial page load**: `to` is the loaded path. This ping is what starts + the visit and counts the entry page view — the document GET alone records + nothing, so bots and admin browsing never register. Reloads are not + visits: the ping is skipped (PerformanceNavigationTiming `reload`), so a + refresh neither counts a second view nor logs a self-transition. The GET + handler only stashes a cross-origin https `Referer` (origin part only) in + an in-memory IP → referer table, consumed by the ping that starts the + visit; internal or absent referers never touch the table. +- **Internal fetch-navigations**: `to` is the target path, sent only after + the swap actually happened (a failed swap falls back to a full load, + whose initial ping counts the view instead — no gap, no double count). +- **External links** (`https` only): `to` is the link's origin. This is the + exit-link record; the user may continue navigating afterwards (new tab, + back), so the exit origin is not necessarily the last trail entry. +- **Excluded**: back/forward (popstate) navigations, and everything while + the user is known to be an admin *and SSO is actually in use* — with no + auth proxy (dev/test) "admin" is everyone's state, so the gate is off and + everything is recorded — or has the editor open (`body.editing`) or the + analytics view open (`body.analytics-open`) — admin noise, not visits. +- The server validates `to`: internal paths must be valid slug paths + ("/" or `[a-z0-9_-]` segments), external ones are re-derived to the + https origin and accepted only when the client sent exactly that. + +## Visits and sessions + +There are no cookies. A visit is tied together by the (IP, User-Agent) pair +(IP from the first `X-Forwarded-For` hop — we sit behind a proxy — else the +direct peer): the first ping from a pair starts a new visit, subsequent +pings extend it. Pings arriving with no known session (server restart) +start a fresh visit from the first ping — treated as missing data rather +than dropped. The (IP, UA) → visit map and the IP → entry-referer table +are in-memory only; IPs are never persisted. + +Each `Visit` record: + +- `start` — timestamp of the first event, +- `entry` — first page (path) seen, +- `referer` — external https origin of the initial load, `""` for direct, +- `trail` — everything seen afterwards in first-seen order: page paths and + external exit origins. Re-visiting an already seen page (incl. the entry) + does not append. + +## Aggregates + +- `transitions`: sparse nested dict `from -> to -> count`. `from` is the + referer origin or `"(direct)"` for initial loads, a page path for pings. +- `views`: time series of page loads, `path -> bucket -> count`, sparse: only + non-zero 5-minute buckets exist (bucket key is its floored ISO timestamp). + Every load counts, including repeats within a visit; external exit origins + are not page views and are not counted here. +- `site_visits`: `bucket -> count` of new visits started, same sparse + 5-minute bucketing. + +Sparseness keeps quiet sites small; dropping old data is a matter of deleting +list/dict entries (`visits` is a plain append-only list, buckets plain keys). + +## Persistence + +The whole `Analytics` struct is JSON-encoded and written atomically +(temp file + rename) on every recorded event. Traffic on a small CMS makes +this cheap enough; batching can be added later without changing the format. + +## Viewing + +The 📊 pen in the banner corner (admins only, injected by pagerite.js next to +the edit pens) opens `AnalyticsView.vue` — a true full-screen app, not an +overlay: `body.analytics-open` hides the page chrome and the document itself +scrolls the view, styled by the active theme's variables. It is addressable +by URL: `#/analytics/` (`week` default; opening via the pen pushes a +history entry so the back button exits, and pagerite.js auto-opens it on +load for editors when the hash is present, so refresh and link sharing work). + +Charts are SVG curves (Catmull-Rom over an edge-aware adaptive Gaussian — +a change-point detector splits the series at traffic-level shifts, then +each segment is smoothed with a bandwidth that ramps with a broad pilot +estimate of the local rate: isolated events stay narrow (~0.4-unit sigma, +peaking at ~1 event/unit), busy traffic widens to a 1-unit sigma. The raw +series is drawn faint underneath). Values are +**per-unit rates** — per hour on the week view (5-minute bucket counts × 12, +plotted at native 5-minute resolution), per day on the month+ ranges — and +the smoothing time scale follows the unit: the month+ sigmas are 24× the +hourly ones. The y max is derived from the smoothed curves so single-bucket +spikes don't blow up the scale, and raw spikes are clamped into the plot. +Axes always start at 0 and end at a multiple of a 1-2-5 major step (max 5 +labeled intervals, minor lines at fifths when integral; the floor is 1/h). +The week range is aligned to Monday 00:00 UTC and overlays up to 8 previous +weeks in the same accent color at decreasing opacity (the current week is +truncated at the current bucket, never drawing fake zeroes for the future); +its x labels are weekday names centered at midday UTC, without vertical grid +lines (day boundaries would be misleading in the viewer's timezone). The +month view labels days the same lineless way — day numbers at noon UTC, +with the month name substituted for the 1st. Year and all are rolling +windows ending at now, re-bucketed to daily points, with boundary lines at +months/years. Below the charts: a radial **transition map** (all pages from +`/_api/pages` — front page at the center, each slug level on its own ring, +siblings clockwise in navigation order from the top, radial gap equal to +the arc spacing — opposite transition directions joined into organic +tapered connections whose middle width is the total count over the full +recorded timescale; internal navigation only for now), per-page view +counts, the top transitions and the 50 most recent visit trails. Data comes from `GET /_api/analytics`, which +returns the raw JSON file contents. diff --git a/frontend/src/AnalyticsView.vue b/frontend/src/AnalyticsView.vue new file mode 100644 index 0000000..c9149a3 --- /dev/null +++ b/frontend/src/AnalyticsView.vue @@ -0,0 +1,222 @@ + + + + + + + diff --git a/frontend/src/TransitionGraph.vue b/frontend/src/TransitionGraph.vue new file mode 100644 index 0000000..e106fd9 --- /dev/null +++ b/frontend/src/TransitionGraph.vue @@ -0,0 +1,100 @@ + + + + + diff --git a/frontend/src/VisitorCharts.vue b/frontend/src/VisitorCharts.vue new file mode 100644 index 0000000..bd7251f --- /dev/null +++ b/frontend/src/VisitorCharts.vue @@ -0,0 +1,189 @@ + + + + + diff --git a/frontend/src/analytics/chart.js b/frontend/src/analytics/chart.js new file mode 100644 index 0000000..9f3b594 --- /dev/null +++ b/frontend/src/analytics/chart.js @@ -0,0 +1,333 @@ +/** + * Chart geometry, smoothing, and SVG path generation for analytics charts. + * + * Fixed 720x180 viewBox, stretched to the panel width; values are per-unit + * rates (hour on the week view, day on month+). + */ + +import { DAY, HOUR, WEEK, mondayUTC } from './time.js' + +export const CHART_W = 720 +export const CHART_H = 180 +export const PAD_TOP = 14 // room above the highest point + +/** + * Y always starts at 0; the max is a multiple of a 1-2-5 major step with at + * most 5 intervals, so labeled ticks are always round and evenly divided. + * Values are per-unit rates, so small scales are legitimate (a lone visit + * smoothes to well under 1/unit) — the floor is 1, not 10. Minor lines + * subdivide each major step in five when that yields integers. + */ +export function yScale(maxValue) { + let step = 1 + outer: for (let exp = -3; exp < 8; exp++) { + for (const base of [1, 2, 5]) { + step = base * 10 ** exp + if (Math.ceil(maxValue / step) <= 5) break outer + } + } + let max = Math.ceil(maxValue / step) * step + if (max < 1) { + max = 1 + step = 0.5 + } + const minor = step >= 5 && step % 5 === 0 ? step / 5 : null + return { max, step, minor } +} + +/** + * Edge-aware adaptive Gaussian smoothing. A change-point detector first + * finds traffic-level shifts (two-unit totals compared on both sides of + * each bucket; strong ratio + significance marks a candidate, and each run + * of candidates keeps only its best-scoring bucket as an edge). Each + * edge-delimited segment is then smoothed independently: a broad two-unit + * pilot estimates the local traffic rate, which ramps the Gaussian sigma + * from ~0.4 units (isolated events stay narrow, peaking at ~1 event/unit) + * up to 1 unit (busy traffic gets full smoothing), and every bucket spreads + * its count with its local sigma, clipped to the segment and renormalized + * so total visitor count is preserved exactly. The unit is one hour on the + * week view and one day on the month+ views, so the smoothing time scale + * follows the range (month+ sigmas are 24x the hourly ones). The raw series + * is drawn faintly behind the curve for reference. Operates on raw counts. + */ +export function smooth(counts, binMinutes, unitMinutes, { + minSigmaMinutes = unitMinutes / Math.sqrt(2 * Math.PI), + maxSigmaMinutes = unitMinutes, + pilotSigmaMinutes = 2 * unitMinutes, + detectorWindowMinutes = 2 * unitMinutes, + // Count thresholds are defined per hour and scale with the unit, so + // "low traffic" means the same thing on hourly and daily views + // (5-20 events/hour = 120-480/day on the month+ ranges). + highTrafficEvents = 10 * unitMinutes / 60, + minRatio = 2.5, + minSignificance = 4, + sigmaRampStart = 5 * unitMinutes / 60, + sigmaRampEnd = 20 * unitMinutes / 60, +} = {}) { + const n = counts.length + if (!n) return counts + const detectorWindowBins = Math.max(1, Math.round(detectorWindowMinutes / binMinutes)) + + const cumsum = new Float64Array(n + 1) + for (let i = 0; i < n; i++) cumsum[i + 1] = cumsum[i] + counts[i] + + // Detect abrupt regime changes from aggregated traffic on both sides. + // Individual bins are deliberately ignored because even high traffic + // produces many 0-1 count bins at five-minute resolution. + const score = new Float64Array(n) + const candidate = new Uint8Array(n) + for (let i = detectorWindowBins; i < n - detectorWindowBins; i++) { + const left = cumsum[i] - cumsum[i - detectorWindowBins] + const right = cumsum[i + detectorWindowBins] - cumsum[i] + const high = Math.max(left, right) + const low = Math.min(left, right) + if (high < highTrafficEvents) continue + const ratio = (high + 1) / (low + 1) + const significance = (high - low) / Math.sqrt(high + low + 1) + if (ratio >= minRatio && significance >= minSignificance) { + candidate[i] = 1 + score[i] = significance * Math.log(ratio) + } + } + + // Collapse each continuous detector region to its strongest boundary. + const edges = [] + for (let i = 0; i < n;) { + if (!candidate[i]) { i++; continue } + let j = i + 1 + while (j < n && candidate[j]) j++ + let best = i + for (let k = i + 1; k < j; k++) { + if (score[k] > score[best]) best = k + } + edges.push(best) + i = j + } + + const reflectIndex = (i, length) => { + while (i < 0 || i >= length) { + i = i < 0 ? -i - 1 : 2 * length - i - 1 + } + return i + } + + const gaussianFilterReflect = (values, sigmaBins) => { + const length = values.length + const radius = Math.ceil(4 * sigmaBins) + const kernel = new Float64Array(radius * 2 + 1) + let sum = 0 + for (let k = -radius; k <= radius; k++) { + const w = Math.exp(-0.5 * (k / sigmaBins) ** 2) + kernel[k + radius] = w + sum += w + } + for (let i = 0; i < kernel.length; i++) kernel[i] /= sum + const out = new Float64Array(length) + for (let i = 0; i < length; i++) { + let value = 0 + for (let k = -radius; k <= radius; k++) { + value += values[reflectIndex(i + k, length)] * kernel[k + radius] + } + out[i] = value + } + return out + } + + // Process each discontinuity-delimited regime independently so neither + // the pilot nor the final Gaussian can see through a detected boundary. + const bounds = [0, ...edges, n] + const smoothed = new Float64Array(n) + for (let b = 0; b < bounds.length - 1; b++) { + const lo = bounds[b] + const length = bounds[b + 1] - lo + const segment = counts.slice(lo, lo + length) + + // Broad pilot estimates only the generic local traffic level used for + // choosing sigma; it is not the final displayed curve. + const pilot = gaussianFilterReflect(segment, pilotSigmaMinutes / binMinutes) + + // Keep isolated/sparse traffic at the minimum bandwidth through + // sigmaRampStart events, then ramp toward maxSigmaMinutes (thresholds + // are per-hour rates scaled to the unit: low traffic is low traffic + // on every range). + const sigmaMinutes = new Float64Array(length) + for (let i = 0; i < length; i++) { + const ratePerUnit = pilot[i] * unitMinutes / binMinutes + let mix = (ratePerUnit - sigmaRampStart) / (sigmaRampEnd - sigmaRampStart) + mix = Math.sqrt(Math.max(0, Math.min(1, mix))) + sigmaMinutes[i] = minSigmaMinutes + mix * (maxSigmaMinutes - minSigmaMinutes) + } + + // Each input bin spreads its own count using its local sigma. The + // per-bin kernel is renormalized after clipping to the segment, + // preserving total visitor count apart from floating-point error. + for (let j = 0; j < length; j++) { + const count = segment[j] + if (!count) continue + const sigmaBins = sigmaMinutes[j] / binMinutes + const radius = Math.ceil(4 * sigmaBins) + const start = Math.max(0, j - radius) + const end = Math.min(length, j + radius + 1) + let weightSum = 0 + for (let i = start; i < end; i++) { + const d = i - j + weightSum += Math.exp(-0.5 * (d / sigmaBins) ** 2) + } + for (let i = start; i < end; i++) { + const d = i - j + smoothed[lo + i] += count * Math.exp(-0.5 * (d / sigmaBins) ** 2) / weightSum + } + } + } + return [...smoothed] +} + +/** + * Catmull-Rom spline through the (smoothed) points, control points clamped + * to the plot area so the curve can never dip below zero or above the max. + */ +export function spline(pts) { + if (pts.length < 3) { + return `M${pts.map((p) => `${p.x},${p.y}`).join('L')}` + } + const clampY = (y) => Math.min(CHART_H, Math.max(PAD_TOP, y)) + let d = `M${pts[0].x},${pts[0].y}` + for (let i = 0; i < pts.length - 1; i++) { + const p0 = pts[i - 1] || pts[i] + const p1 = pts[i] + const p2 = pts[i + 1] + const p3 = pts[i + 2] || p2 + const c1y = clampY(p1.y + (p2.y - p0.y) / 6) + const c2y = clampY(p2.y - (p3.y - p1.y) / 6) + d += `C${p1.x + (p2.x - p0.x) / 6},${c1y} ` + + `${p2.x - (p3.x - p1.x) / 6},${c2y} ${p2.x},${p2.y}` + } + return d +} + +/** Build a full chart model from a series descriptor produced by time.js. */ +export function buildChart(input) { + if (!input || !input.series.length) return null + const { series, t0, t1, rate, binMinutes, unitMinutes } = input + // Values are per-unit rates (hour on the week view, day on month+); the + // y max is derived from the *smoothed* curves so random single-bucket + // spikes don't blow up the scale. Smoothing works on raw counts (its edge + // detector thresholds are count-based), the result is scaled back to rates. + const smoothed = series.map((s) => + smooth(s.points.map((p) => p.count), binMinutes, unitMinutes).map((v) => v * rate)) + // Scale from the current/primary series only; older overlay weeks are drawn + // with the same scale and allowed to overflow if they are busier. + const highest = Math.max(0, ...smoothed[0]) + const { max, step, minor } = yScale(highest) + const x = (t) => ((t - t0) / (t1 - t0)) * CHART_W + const y = (v) => PAD_TOP + (1 - Math.max(0, v) / max) * (CHART_H - PAD_TOP) + const drawn = series.map((s, si) => { + const pts = s.points.map((p, i) => ({ x: x(p.t), y: y(smoothed[si][i]) })) + const line = spline(pts) + const first = pts[0] + const last = pts.at(-1) + return { + ...s, + line, + area: s.area ? `${line}L${last.x},${CHART_H}L${first.x},${CHART_H}Z` : null, + } + }) + // Major (labeled) and minor (hairline) y grid ticks. + const majors = [] + const minors = [] + const nMajor = Math.round(max / step) + for (let k = 0; k <= nMajor; k++) { + const v = k * step + majors.push({ value: v, y: y(v), bottom: (1 - PAD_TOP / CHART_H) * (v / max) * 100 }) + } + if (minor) { + for (let v = minor; v < max; v += minor) { + if (v % step !== 0) minors.push({ y: y(v) }) + } + } + // X ticks. Week view: weekday labels centered at midday UTC, no vertical + // lines (day boundaries would be misleading in the viewer's timezone). + // Month view: likewise lineless, day numbers at noon UTC with the month + // name substituted for the 1st (marking the month change). Longer + // ranges: boundary lines at Mondays / months / years. + const isWeek = t1 - t0 === WEEK + const isMonth = !isWeek && t1 - t0 <= 31 * DAY + const xticks = isWeek + ? Array.from({ length: 7 }, (_, d) => { + const t = t0 + d * DAY + 12 * HOUR + return { + x: x(t), left: ((t - t0) / (t1 - t0)) * 100, + label: new Date(t).toLocaleDateString(undefined, { + weekday: 'short', timeZone: 'UTC', + }), + line: false, + } + }) + : isMonth + ? Array.from( + { length: Math.floor((t1 - Math.ceil(t0 / DAY) * DAY) / DAY) }, + (_, d) => { + const day = Math.ceil(t0 / DAY) * DAY + d * DAY + const date = new Date(day) + const t = day + 12 * HOUR + return { + x: x(t), left: ((t - t0) / (t1 - t0)) * 100, + label: date.getUTCDate() === 1 + ? date.toLocaleDateString(undefined, { month: 'short', timeZone: 'UTC' }) + : String(date.getUTCDate()), + line: false, + } + }, + ) + : xticksFor(t0, t1).map((t) => ({ + x: x(t), left: ((t - t0) / (t1 - t0)) * 100, + label: fmtTick(t, t1 - t0), line: true, + })) + return { max, majors, minors, series: drawn, xticks } +} + +/** X ticks for year/all: Monday boundaries up to a quarter, UTC month + * boundaries up to a few years, then years. */ +export function xticksFor(t0, t1) { + const span = t1 - t0 + const ticks = [] + if (span <= 100 * DAY) { + for (let t = mondayUTC(t0); t <= t1; t += WEEK) { + if (t >= t0) ticks.push(t) + } + return ticks + } + if (span <= 4 * 365 * DAY) { + const d = new Date(t0) + let t = Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 1) + for (; t <= t1; ) { + ticks.push(t) + const m = new Date(t) + t = Date.UTC(m.getUTCFullYear(), m.getUTCMonth() + 1, 1) + } + return ticks + } + const d = new Date(t0) + for (let yr = d.getUTCFullYear() + 1; Date.UTC(yr, 0, 1) <= t1; yr++) { + ticks.push(Date.UTC(yr, 0, 1)) + } + return ticks +} + +export function fmtTick(t, span) { + const d = new Date(t) + if (span <= 100 * DAY) { + return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric', timeZone: 'UTC' }) + } + if (span <= 4 * 365 * DAY) { + return d.getUTCMonth() === 0 + ? d.toLocaleDateString(undefined, { year: 'numeric', timeZone: 'UTC' }) + : d.toLocaleDateString(undefined, { month: 'short', timeZone: 'UTC' }) + } + return d.toLocaleDateString(undefined, { year: 'numeric', timeZone: 'UTC' }) +} + +/** Y labels: integers when the step allows, one decimal for fractional steps. */ +export function fmtY(v) { + return Number.isInteger(v) ? String(v) : v.toFixed(1) +} diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js new file mode 100644 index 0000000..f1f6346 --- /dev/null +++ b/frontend/src/analytics/format.js @@ -0,0 +1,54 @@ +/** + * Formatters and aggregators for summary sections: totals and the recent + * visit trail. + */ + +/** Total page views across every page and every bucket. */ +export function calcTotalViews(views) { + let n = 0 + for (const buckets of Object.values(views || {})) { + for (const c of Object.values(buckets)) n += c + } + return n +} + +/** Build a path -> page title lookup from the site tree. */ +function buildTitleMap(pageTree) { + const titles = new Map() + const walk = (items) => { + for (const item of items || []) { + titles.set(`/${item.path}`, item.title) + walk(item.children) + } + } + walk(pageTree) + return titles +} + +/** Last path segment for display; front page becomes a house icon. */ +function slugOf(path) { + return path === '/' ? '🏠' : path.split('/').pop() +} + +/** + * Format recent visits for display, newest first. Each step is a linked slug + * pointing to its article; external referers/origins and direct entries are + * omitted. The link title shows the article heading when known. + */ +export function formatRecentVisits(visits, pageTree, limit = 50) { + const titles = buildTitleMap(pageTree) + return [...visits] + .reverse() + .map((v) => ({ + when: new Date(v.start).toLocaleString(), + steps: [v.entry, ...(v.trail || [])] + .filter((p) => p?.startsWith('/')) + .map((p) => ({ + path: p, + slug: slugOf(p), + title: titles.get(p) || '', + })), + })) + .filter((v) => v.steps.length) + .slice(0, limit) +} diff --git a/frontend/src/analytics/time.js b/frontend/src/analytics/time.js new file mode 100644 index 0000000..7d11bc9 --- /dev/null +++ b/frontend/src/analytics/time.js @@ -0,0 +1,141 @@ +/** + * Time ranges, week alignment and re-bucketing for analytics charts. + * + * Raw data comes as sparse 5-minute buckets; the range picks the x window + * and a coarser bucket size to keep point counts sane. The week range is + * aligned to Monday 00:00 UTC and overlays previous weeks' curves (fading + * with age), so weekly patterns compare directly. + */ + +export const MIN5 = 5 * 60e3 +export const HOUR = 3600e3 +export const DAY = 86400e3 +export const WEEK = 7 * DAY + +export const RANGES = { + week: { label: 'week' }, + month: { label: 'month', span: 30 * DAY, bucket: 6 * HOUR }, + year: { label: 'year', span: 365 * DAY, bucket: DAY }, + all: { label: 'all', span: null, bucket: DAY }, +} + +/** Monday 00:00 UTC of the week containing t (epoch day 0 was a Thursday). */ +export function mondayUTC(t) { + const d = Math.floor(t / DAY) + return (d - ((d + 3) % 7)) * DAY +} + +/** Parse sparse timestamp buckets into a { epochMs: count } map. */ +export function rawTimes(buckets) { + const raw = {} + // Key by parsed timestamp: Python writes "+00:00", JS ISO uses "Z". + for (const [k, c] of Object.entries(buckets || {})) raw[Date.parse(k)] = c + return raw +} + +/** Sum counts from raw 5-minute buckets between t0 (inclusive) and t1 (exclusive). */ +export function sumRange(raw, t0, t1) { + let n = 0 + for (let s = t0; s < t1; s += MIN5) n += raw[s] || 0 + return n +} + +/** + * One series per overlaid week: [this week, 1 week ago, ...], at native + * 5-minute resolution, up to 8 weeks back (and only weeks that overlap the + * recorded data at all). The current week is truncated at the current bucket + * — no fake zeroes drawn for the future. Counts are rates per hour + * (bucket count * 12): a lone visit in a 5-minute bucket reads as "12/h". + * The coarser ranges use per-day rates instead (unitMinutes = 24*60). + */ +export function weeklySeries(buckets) { + const raw = rawTimes(buckets) + const times = Object.keys(raw).map(Number) + if (!times.length) return null + const now = Date.now() + const thisMonday = mondayUTC(now) + const oldest = Math.min(...times) + // Weeks back as far as the data reaches: difference in Monday indices. + const available = (thisMonday - mondayUTC(oldest)) / WEEK + 1 + const count = Math.min(available, 8) + const out = [] + for (let back = 0; back < count; back++) { + const start = thisMonday - back * WEEK + const end = back === 0 + ? Math.min(start + WEEK, Math.floor(now / MIN5) * MIN5 + MIN5) + : start + WEEK + const points = [] + for (let t = start; t < end; t += MIN5) { + points.push({ t, count: raw[t] || 0 }) + } + out.push({ + points, + label: back === 0 ? 'this week' : `${back}w ago`, + opacity: Math.max(0.15, 1 - back * 0.25), + area: back === 0, + }) + } + return { + series: out, + t0: thisMonday, + t1: thisMonday + WEEK, + rate: HOUR / MIN5, + binMinutes: 5, + unitMinutes: 60, + unit: 'hour', + } +} + +/** + * Rolling window for the non-week ranges (x max = now), counts converted + * to per-day rates (the unit the month+ charts are read in). + */ +export function rollingSeries(buckets, rangeKey) { + const raw = rawTimes(buckets) + const times = Object.keys(raw).map(Number) + if (!times.length) return null + const { span, bucket } = RANGES[rangeKey] + const t1 = Math.floor(Date.now() / bucket) * bucket + bucket + const t0 = span != null + ? t1 - span + : Math.floor(Math.min(...times) / bucket) * bucket + const points = [] + for (let t = t0; t < t1; t += bucket) { + points.push({ t, count: sumRange(raw, t, t + bucket) }) + } + return { + series: [{ points, label: '', opacity: 1, area: true }], + t0, + t1, + rate: DAY / bucket, + binMinutes: bucket / 60e3, + unitMinutes: 24 * 60, + unit: 'day', + } +} + +/** Dispatch to weekly or rolling series based on the selected range. */ +export function makeSeries(buckets, rangeKey) { + return rangeKey === 'week' + ? weeklySeries(buckets) + : rollingSeries(buckets, rangeKey) +} + +/** + * Absolute UTC time window for a given range key. Used to filter visits, + * transitions and views to the same period the charts are showing. + * Returns { t0, t1 } where null means unbounded. + */ +export function rangeWindow(rangeKey) { + const now = Date.now() + if (rangeKey === 'week') { + const start = mondayUTC(now) + return { t0: start, t1: start + WEEK } + } + if (rangeKey === 'all') { + return { t0: null, t1: null } + } + const { span, bucket } = RANGES[rangeKey] + const t1 = Math.floor(now / bucket) * bucket + bucket + return { t0: t1 - span, t1 } +} diff --git a/frontend/src/analytics/transitions.js b/frontend/src/analytics/transitions.js new file mode 100644 index 0000000..4268a48 --- /dev/null +++ b/frontend/src/analytics/transitions.js @@ -0,0 +1,394 @@ +/** + * Radial transition map and helpers. + * + * Radial site map: the front page at the center, each slug level on its own + * ring. All pages of the site are shown (from /_api/pages), plus any extra + * paths seen in transitions (deleted pages); siblings run clockwise in + * navigation order, starting at the top. Internal path -> path transitions + * join opposite directions into straight connections (middle width = total + * count, wrapping the node circles at both ends); external referers/exits + * are not shown (yet). Self-loops (reload pings) are also skipped. + */ + +export const TNODE_R = 34 // node circles hold the slug and the view count + +/** Flatten the site tree into navigation order via DFS. */ +function buildNavigationOrder(pageTree) { + const order = new Map() + const walk = (items) => { + for (const item of items || []) { + const p = `/${item.path}` + if (!order.has(p)) order.set(p, order.size) + walk(item.children) + } + } + walk(pageTree) + return order +} + +/** Map page paths to their article titles from the site tree. */ +function buildTitleMap(pageTree) { + const titles = new Map() + const walk = (items) => { + for (const item of items || []) { + titles.set(`/${item.path}`, item.title) + walk(item.children) + } + } + walk(pageTree) + return titles +} + +/** Extract internal page-to-page transitions, excluding self-loops. */ +function collectInternalTransitions(transitions) { + const internal = [] + for (const [fr, tos] of Object.entries(transitions || {})) { + if (!fr.startsWith('/')) continue + for (const [to, count] of Object.entries(tos)) { + if (to.startsWith('/') && to !== fr) internal.push({ fr, to, count }) + } + } + return internal +} + +/** Build nodes with depth and a path lookup map; children are wired to parents. */ +function buildNodeTree(internal, navOrder) { + const paths = new Set(['/', ...navOrder.keys()]) + for (const e of internal) { paths.add(e.fr); paths.add(e.to) } + + const depth = (p) => (p === '/' ? 0 : p.split('/').length - 1) + const nodes = [...paths].map((p) => ({ + path: p, depth: depth(p), angle: 0, children: [], + })) + const byPath = new Map(nodes.map((n) => [n.path, n])) + + // Parent is the nearest ancestor present in the map, front page last. + const parentOf = (p) => { + let q = p + while (q !== '/') { + q = q.slice(0, q.lastIndexOf('/')) || '/' + if (byPath.has(q)) return byPath.get(q) + } + return byPath.get('/') + } + for (const n of nodes) { + if (n.path !== '/') parentOf(n.path).children.push(n) + } + + return { nodes, byPath, root: byPath.get('/') } +} + +/** Sort children by navigation order and compute each subtree's angular weight. */ +function prepareWeights(root, navOrder) { + const byNav = (a, b) => + (navOrder.get(a.path) ?? Infinity) - (navOrder.get(b.path) ?? Infinity) + || a.path.localeCompare(b.path) + const weight = (n) => + n.children.length ? n.children.reduce((s, k) => s + weight(k), 0) : 1 / n.depth + + const walkSort = (n) => { + n.children.sort(byNav) + n.children.forEach(walkSort) + } + walkSort(root) + + return weight +} + +/** Assign angles clockwise starting from the top (-PI/2). */ +function layoutAngles(root, unit, weight) { + const lay = (n, a0) => { + n.angle = a0 + let a = a0 + for (const k of n.children) { + lay(k, a) + a += weight(k) * unit + } + } + let a = -Math.PI / 2 + for (const k of root.children) { + lay(k, a) + a += weight(k) * unit + } +} + +/** Compute radial positions, view counts and labels for each node. */ +function positionNodes(nodes, maxDepth, unit, viewsData, titles) { + // Constant radial gap between rings, equal to the arc spacing of nodes + // along a ring: leaf arc = unit * GAP, so GAP scales up with `unit` on + // sparse trees (where closing the circle forces wider arcs) and with + // 1/unit on dense ones (keeping arcs at the node clearance). + const CLEAR = 2 * TNODE_R + 12 + const GAP = CLEAR * Math.max(unit, 1 / unit) + const radius = (d) => d * GAP + + const viewCount = (p) => { + let n = 0 + for (const c of Object.values(viewsData?.[p] || {})) n += c + return n + } + + for (const n of nodes) { + const r = radius(n.depth) + n.x = Math.cos(n.angle) * r + n.y = Math.sin(n.angle) * r + n.views = viewCount(n.path) + // Slug inside the circle; full title goes on the link title attribute. + const slug = n.path === '/' ? '🏠' : n.path.split('/').pop() + n.label = slug.length > 11 ? `${slug.slice(0, 10)}…` : slug + n.title = titles.get(n.path) || '' + } + + return { radius, GAP } +} + +/** + * Family structure at a glance: a radial spoke from each parent to its + * first child, and a ring arc across each sibling group from first to last + * child in navigation (clockwise) order. + */ +function buildFamilyArcs(nodes, radius) { + const arcs = [] + for (const n of nodes) { + if (!n.children.length) continue + // The spoke aims along the FIRST CHILD's angle (the node's own angle + // coincides with it, except for the center page which has none). + const first = n.children[0] + const r1 = radius(n.depth) + TNODE_R + const r2 = radius(first.depth) - TNODE_R + arcs.push({ + d: `M ${Math.cos(first.angle) * r1} ${Math.sin(first.angle) * r1} ` + + `L ${Math.cos(first.angle) * r2} ${Math.sin(first.angle) * r2}`, + }) + if (n.children.length < 2) continue + const r = radius(n.children[0].depth) + const a0 = n.children[0].angle + const a1 = n.children[n.children.length - 1].angle + if (a1 - a0 >= 2 * Math.PI - 1e-6) continue // full circle: degenerate arc + const large = a1 - a0 > Math.PI ? 1 : 0 + arcs.push({ + d: `M ${Math.cos(a0) * r} ${Math.sin(a0) * r} ` + + `A ${r} ${r} 0 ${large} 1 ${Math.cos(a1) * r} ${Math.sin(a1) * r}`, + }) + } + return arcs +} + +/** Collapse opposite transition directions into one unordered pair per page pair. */ +function aggregatePairs(internal) { + const pairs = new Map() // unordered pair key -> [countAB, countBA] + for (const e of internal) { + const forward = e.fr < e.to + const k = forward ? `${e.fr} ${e.to}` : `${e.to} ${e.fr}` + const c = pairs.get(k) || [0, 0] + c[forward ? 0 : 1] += e.count + pairs.set(k, c) + } + return pairs +} + +const fmtPt = (p) => `${p[0].toFixed(2)} ${p[1].toFixed(2)}` + +/** Build one ribbon edge between two nodes with counts ab and ba. */ +function buildRibbon(a, b, ab, ba) { + const count = ab + ba + const len = Math.hypot(b.x - a.x, b.y - a.y) || 1 + const ux = (b.x - a.x) / len + const uy = (b.y - a.y) / len + const nx = -uy + const ny = ux + + // Half-width of the thin middle and radius of the node surround. + const wMid = 0.75 + 6.75 * (Math.min(count, 100) / 100) ** 1.5 + const R2 = TNODE_R + 3 + + // Attachment points sit somewhat forward from the side of the node, + // leaving enough room for the surround to flow naturally into the flare. + const BETA = (65 * Math.PI) / 180 + const END = R2 * Math.cos(BETA) + const wEnd = R2 * Math.sin(BETA) + + // Fixed flare length, clamped so the two ends cannot overlap. + const FLARE = Math.min(36, Math.max(0, (len - 2 * END) / 2)) + + // Point on the connection centerline at distance t from A, offset s + // perpendicular to it. + const P = (t, s) => [ + a.x + t * ux + s * nx, + a.y + t * uy + s * ny, + ] + + // Arc around a node from p to q the long way, passing its back side. + const wrap = (p, q, node, back) => { + const ang = (pt2) => + Math.atan2(pt2[1] - node[1], pt2[0] - node[0]) + + const TAU = 2 * Math.PI + const da = ((ang(back) - ang(p)) % TAU + TAU) % TAU + const db = ((ang(q) - ang(p)) % TAU + TAU) % TAU + + return `A ${R2} ${R2} 0 1 ${da < db ? 1 : 0} ${fmtPt(q)} ` + } + + // Build one side of a flare in node -> middle order. + const flarePoints = (endT, midT, s, dir) => { + const span = Math.abs(midT - endT) + const pEnd = P(endT, s * wEnd) + const pMid = P(midT, s * wMid) + + // At the node, leave tangent to the circular surround. + // The circle radius at the attachment is locally: + // A: (+END, ±wEnd) + // B: (-END, ±wEnd) + // A perpendicular tangent pointing into the connection therefore has + // these centerline/normal components. + const tangentT = dir * wEnd / R2 + const tangentS = -s * END / R2 + + const hEnd = span * 0.65 + const hMid = span * 0.4 + + const cEnd = P( + endT + tangentT * hEnd, + s * wEnd + tangentS * hEnd, + ) + + // At the thin end, arrive parallel with the centerline. + const cMid = P( + midT - dir * hMid, + s * wMid, + ) + + return { pEnd, cEnd, cMid, pMid } + } + + // Emit a cubic in either traversal direction. Reversing a cubic requires + // swapping its control points, rather than recalculating the geometry. + const curve = (f, reverse = false) => { + if (!reverse) { + return `C ${fmtPt(f.cEnd)} ${fmtPt(f.cMid)} ${fmtPt(f.pMid)} ` + } + return `C ${fmtPt(f.cMid)} ${fmtPt(f.cEnd)} ${fmtPt(f.pEnd)} ` + } + + const LA = P(END, wEnd) + const RA = P(END, -wEnd) + const LB = P(len - END, wEnd) + const RB = P(len - END, -wEnd) + + const aLeft = flarePoints(END, END + FLARE, 1, 1) + const bLeft = flarePoints(len - END, len - END - FLARE, 1, -1) + const bRight = flarePoints(len - END, len - END - FLARE, -1, -1) + const aRight = flarePoints(END, END + FLARE, -1, 1) + + const d = `M ${fmtPt(LA)} ` + + curve(aLeft) + + `L ${fmtPt(bLeft.pMid)} ` + + curve(bLeft, true) + + wrap(LB, RB, [b.x, b.y], P(len + R2, 0)) + + curve(bRight) + + `L ${fmtPt(aRight.pMid)} ` + + curve(aRight, true) + + wrap(RA, LA, [a.x, a.y], P(-R2, 0)) + + 'Z' + + return { + d, + title: `${a.path} ↔ ${b.path}: ${count} (${ab} / ${ba})`, + } +} + +/** Build ribbon edges for every aggregated page-to-page pair. */ +function buildRibbonEdges(pairs, byPath) { + return [...pairs].map(([k, [ab, ba]]) => { + const [pf, pt] = k.split(' ') + const a = byPath.get(pf) + const b = byPath.get(pt) + return buildRibbon(a, b, ab, ba) + }) +} + +/** Parse a visit start timestamp, which may already be numeric or an ISO string. */ +function visitStart(v) { + return typeof v.start === 'number' ? v.start : Date.parse(v.start) +} + +/** + * Derive internal path -> path transitions from the visits list, optionally + * restricted to a time window. This is the only time-filterable source of + * transitions (the server-side aggregate has no per-transition timestamps). + * Re-visits within the same visit are not recorded in `trail`, so this yields + * first-seen navigation chains rather than every ping. + */ +export function buildTransitionsFromVisits(visits, t0, t1) { + const transitions = {} + for (const v of visits || []) { + const start = visitStart(v) + if ((t0 != null && start < t0) || (t1 != null && start >= t1)) continue + const path = [v.entry, ...(v.trail || [])].filter((p) => p?.startsWith('/')) + for (let i = 0; i < path.length - 1; i++) { + const fr = path[i] + const to = path[i + 1] + if (fr === to) continue + transitions[fr] = transitions[fr] || {} + transitions[fr][to] = (transitions[fr][to] || 0) + 1 + } + } + return transitions +} + +/** Keep only the 5-minute view buckets that fall inside [t0, t1). */ +export function filterViewsByRange(views, t0, t1) { + const filtered = {} + for (const [path, buckets] of Object.entries(views || {})) { + const out = {} + for (const [k, c] of Object.entries(buckets)) { + const t = Date.parse(k) + if ((t0 == null || t >= t0) && (t1 == null || t < t1)) out[k] = c + } + if (Object.keys(out).length) filtered[path] = out + } + return filtered +} + +/** + * Build the radial transition map model. + * Returns { nodes, edges, arcs, r } or null when there is nothing to show. + */ +export function buildTransitionGraph(data, pageTree) { + const internal = collectInternalTransitions(data?.transitions) + const navOrder = buildNavigationOrder(pageTree) + const titles = buildTitleMap(pageTree) + + if (!internal.length && !navOrder.size) return null + + const { nodes, byPath, root } = buildNodeTree(internal, navOrder) + const weightFn = prepareWeights(root, navOrder) + const unit = (2 * Math.PI) / weightFn(root) + layoutAngles(root, unit, weightFn) + + const maxDepth = Math.max(1, ...nodes.map((n) => n.depth)) + const { radius, GAP } = positionNodes(nodes, maxDepth, unit, data?.views, titles) + const arcs = buildFamilyArcs(nodes, radius) + const pairs = aggregatePairs(internal) + const edges = buildRibbonEdges(pairs, byPath) + + // Tight bounding box of the actual nodes; edges and arcs stay within the + // node circles, so node bounds plus node radius are sufficient. + const pad = 16 + const xs = nodes.map((n) => n.x) + const ys = nodes.map((n) => n.y) + const bounds = { + x0: Math.min(...xs) - TNODE_R - pad, + y0: Math.min(...ys) - TNODE_R - pad, + x1: Math.max(...xs) + TNODE_R + pad, + y1: Math.max(...ys) + TNODE_R + pad, + } + + return { + nodes, + edges, + arcs, + bounds, + } +} diff --git a/frontend/src/assets/pagerite.css b/frontend/src/assets/pagerite.css index 385937e..66f54b4 100644 --- a/frontend/src/assets/pagerite.css +++ b/frontend/src/assets/pagerite.css @@ -596,7 +596,7 @@ blockquote p { background: color-mix(in srgb, var(--admonition-color, var(--accent)) 7%, transparent); } -.admonition > :last-child { +.admonition> :last-child { margin-bottom: 0; } @@ -698,12 +698,6 @@ td { transparent 75%); } -tbody tr:nth-child(even) td { - background: linear-gradient(160deg, - color-mix(in srgb, var(--table-tint, var(--accent)) 9%, transparent), - color-mix(in srgb, var(--table-tint, var(--accent)) 3%, transparent) 75%); -} - tbody tr+tr td { border-top: 1px solid color-mix(in srgb, var(--table-tint, var(--accent)) 12%, transparent); } diff --git a/frontend/src/main.js b/frontend/src/main.js index c89f5b6..2e3d31f 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -17,6 +17,7 @@ if (import.meta.env.DEV) { import { createApp } from 'vue' import EditorShell from './EditorShell.vue' +import AnalyticsView from './AnalyticsView.vue' let host = null let app = null @@ -96,3 +97,52 @@ export function closeEditor() { if (!visible && restoreTitle != null) document.title = restoreTitle }) } + +// --- Full-screen analytics app --------------------------------------------- +// Replaces the page chrome while open (body.analytics-open hides it, see +// AnalyticsView.vue); opened from the 📊 pen or directly via the URL hash +// #/analytics/ so refresh and link sharing stay in analytics. +let analyticsHost = null +let analyticsApp = null + +function analyticsHashRange() { + const m = location.hash.match(/^#\/analytics(?:\/(\w+))?/) + return m ? m[1] || 'week' : null +} + +function onHashChange() { + if (analyticsHashRange() === null) closeAnalytics() + else openAnalytics() +} + +export function openAnalytics() { + if (analyticsHost) return + let r = analyticsHashRange() + if (r === null) { + r = 'week' + // Pushed (not replaced) so the back button exits the app via hashchange. + history.pushState(null, '', `#/analytics/${r}`) + } + analyticsHost = document.createElement('div') + document.body.append(analyticsHost) + document.body.classList.add('analytics-open') + analyticsApp = createApp(AnalyticsView, { + initialRange: r, + onClose: closeAnalytics, + }) + analyticsApp.mount(analyticsHost) + addEventListener('hashchange', onHashChange) +} + +export function closeAnalytics() { + if (!analyticsHost) return + removeEventListener('hashchange', onHashChange) + analyticsApp?.unmount() + analyticsApp = null + analyticsHost?.remove() + analyticsHost = null + document.body.classList.remove('analytics-open') + if (analyticsHashRange() !== null) { + history.replaceState(null, '', location.pathname + location.search) + } +} diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js index 1df2815..01f0b81 100644 --- a/frontend/src/pagerite.js +++ b/frontend/src/pagerite.js @@ -106,6 +106,16 @@ import "overlayscrollbars/overlayscrollbars.css"; if (canEdit) { pens.append(makePen("banner")); pens.append(makePen("site")); + if (editorMeta) { + // Full-screen analytics view (separate from the docked panel). + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "edit-link analytics-link"; + btn.title = "analytics"; + btn.textContent = "📊"; + btn.dataset.editorSrc = editorMeta.src; + pens.append(btn); + } } if (ssoAvailable) pens.append(makeAuthLink(isAdmin)); banner.after(pens); @@ -115,7 +125,7 @@ import "overlayscrollbars/overlayscrollbars.css"; async function setupAuth() { const src = document.querySelector('meta[name="pagerite:editor-src"]')?.content; - if (!src) return; + if (!src) { pingEntryOnce(); return; } editorMeta = { src, css: document.querySelector('meta[name="pagerite:editor-css"]')?.content, @@ -138,6 +148,19 @@ import "overlayscrollbars/overlayscrollbars.css"; } renderAuthUi(); + pingEntryOnce(); + + // The analytics app is addressable by URL (#/analytics/), so a + // refresh or a shared link lands back in it. Only for editors. + const openAnalyticsFromHash = () => { + if (!location.hash.startsWith("#/analytics")) return; + if (!(isAdmin || !ssoAvailable) || !editorMeta) return; + import(/* @vite-ignore */ editorMeta.src) + .then((m) => m.openAnalytics()) + .catch((e) => console.error("analytics view load failed:", e)); + }; + openAnalyticsFromHash(); + addEventListener("hashchange", openAnalyticsFromHash); } // Returning to the page via history back/forward may restore a cached @@ -326,6 +349,42 @@ import "overlayscrollbars/overlayscrollbars.css"; }, { passive: true }); } + // --- Analytics pings --------------------------------------------------- + // Fire-and-forget POST /_a {fr, to}: on the initial page load (starts the + // visit — the server counts nothing from the document GET alone), for + // internal fetch-navigations and for external https exits. Excluded: + // back/forward (popstate never pings) and everything while we know the + // user is an admin — but only when SSO is actually in use; with no auth + // (dev/test) "admin" is everyone's state and nothing would be recorded — + // or has the editor/analytics view open (admin noise, not visits). + // See docs/analytics.md. + function ping(to, fr = currentPath) { + if ((ssoAvailable && isAdmin) || document.body.classList.contains("editing") + || document.body.classList.contains("analytics-open")) return; + try { + fetch("/_a", { + method: "POST", + keepalive: true, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ fr, to }), + }); + } catch { /* analytics must never break navigation */ } + } + + // The initial page load pings too — it is what starts the visit and + // counts the entry page view (the document GET alone records nothing). + // Sent once per load, after the auth probes so the admin gate applies; + // the pageshow re-probe must not ping again. Reloads are not visits: + // pinging them would double-count the view and log a self-transition. + let entryPinged = false; + function pingEntryOnce() { + if (entryPinged) return; + entryPinged = true; + const nav = performance.getEntriesByType?.("navigation")[0]; + if (nav ? nav.type === "reload" : performance.navigation?.type === 1) return; + ping(currentPath); + } + // --- Fetch navigation ------------------------------------------------ async function load(url, push = true, back = false) { // Navigating with the editor open closes it; unsaved edits are lost @@ -348,12 +407,12 @@ import "overlayscrollbars/overlayscrollbars.css"; doc = new DOMParser().parseFromString(await res.text(), "text/html"); } catch { location.href = url; // fall back to a normal navigation - return; + return false; } } if (REGIONS.some((id) => !doc.getElementById(id))) { location.href = url; - return; + return false; } const doit = () => { for (const id of REGIONS) { @@ -410,6 +469,7 @@ import "overlayscrollbars/overlayscrollbars.css"; currentPath = new URL(finalUrl, location.href).pathname; if (push) history.pushState(null, "", finalUrl); scrollTo(0, 0); + return true; } addEventListener("click", (ev) => { @@ -419,6 +479,16 @@ import "overlayscrollbars/overlayscrollbars.css"; // the Vue app on demand (with any extra styles) and mount it in place. // Clicking the pen of the already-open tab closes the shell; clicking // another pen switches the shell to that tab. + // The 📊 pen opens the full-screen analytics view (its own Vue app, + // not a tab of the docked editor shell). + const analyticsBtn = ev.target.closest("button.analytics-link"); + if (analyticsBtn && analyticsBtn.dataset.editorSrc) { + ev.preventDefault(); + import(/* @vite-ignore */ analyticsBtn.dataset.editorSrc) + .then((m) => m.openAnalytics()) + .catch((e) => console.error("analytics view load failed:", e)); + return; + } const editBtn = ev.target.closest("button.edit-link"); if (editBtn && editBtn.dataset.editorSrc) { ev.preventDefault(); @@ -447,16 +517,27 @@ import "overlayscrollbars/overlayscrollbars.css"; const a = ev.target.closest("a[href]"); if (!a || a.target || a.hasAttribute("download")) return; const url = new URL(a.href, location.href); - if (url.origin !== location.origin) return; + if (url.origin !== location.origin) { + // External link: the browser navigates; just record the exit (https + // origins only, stripped to the origin part server-side anyway). + if (url.protocol === "https:") ping(url.origin); + return; + } // Same-page anchor links (footnotes etc.): let the browser handle them if (url.pathname === location.pathname && url.hash) return; // Machinery and auth endpoints are never fetch-navigated. if (url.pathname.startsWith("/_") || url.pathname.startsWith("/auth")) return; ev.preventDefault(); - load(url); + // Capture the source now: load() updates currentPath before pinging. + const from = currentPath; + load(url).then((ok) => { if (ok) ping(url.pathname, from); }); }); - addEventListener("popstate", () => load(location.href, false, true)); + addEventListener("popstate", () => { + // Hash-only history entries (the analytics app) are not navigations. + if (location.pathname === currentPath) return; + load(location.href, false, true); + }); // --- Task-list checkboxes ------------------------------------------------ // Checkboxes in the rendered article are live: toggling them edits the diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 490d060..8d6cae6 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -9,13 +9,14 @@ const backendUrl = process.env.PAGERITE_BACKEND_URL || 'http://localhost:3200' // Proxy content pages (/slug, /path/to/slug) to the FastAPI backend in dev. // Excludes Vite internals (/@..., /src, /node_modules, /__...) and the -// backend's /_ prefix. /_api and /_f are handled by the fastapi-vue plugin. +// backend's /_ prefix. /_api, /_f, /_themes and the /_a analytics ping are +// handled by the fastapi-vue plugin. const CONTENT_PROXY = '^\\/(?!_|@|src|node_modules|__)(?:[^./?]+(?:\\/[^./?]+)*)?(?:\\?.*)?$' // https://vite.dev/config/ export default defineConfig({ plugins: [ - fastapiVue({ paths: ["/_api", "/_f", "/_themes"] }), + fastapiVue({ paths: ["/_api", "/_f", "/_themes", "/_a"] }), vue(), vueDevTools(), ], diff --git a/pagerite/analytics.py b/pagerite/analytics.py new file mode 100644 index 0000000..6b64cd7 --- /dev/null +++ b/pagerite/analytics.py @@ -0,0 +1,174 @@ +"""Server-side visit analytics (collection only; see docs/analytics.md). + +Events come from navigation pings POSTed to /_a by pagerite.js: the first +ping on page load starts a visit, later pings extend it, and pings with no +known session start a fresh one (missing data, not dropped). The document +GET handler only stashes the entry referer (external https origin) in an +in-memory IP -> referer table, consumed when the ping starts the visit; +nothing is counted without a ping (bots and admin browsing stay invisible). +The session map is in-memory only; IPs are never persisted. + +Data is a msgspec Struct JSON-dumped to its own file (not the kanta db), +rewritten atomically on every recorded event. +""" + +import os +import re +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlparse + +import msgspec + + +class Visit(msgspec.Struct, omit_defaults=True): + """One visit: the initial-load data plus everything seen afterwards. + + ``trail`` holds page paths and external exit origins in first-seen + order; re-visiting an already seen page does not append. The entry + page itself is in ``entry``, not in the trail. + """ + + start: datetime + entry: str + #: External https origin of the initial load, "" for direct visits. + referer: str = "" + trail: list[str] = [] + + +class Analytics(msgspec.Struct, omit_defaults=True): + """Root of the analytics JSON file. Append-only by design: old data is + dropped by deleting list entries / bucket keys.""" + + visits: list[Visit] = [] + #: Page transition matrix: from -> to -> count. ``from`` is the referer + #: origin or "(direct)" for initial loads, a page path for pings. + transitions: dict[str, dict[str, int]] = {} + #: Page views per 5-minute bucket: path -> bucket ISO -> count (sparse). + views: dict[str, dict[str, int]] = {} + #: New visits per 5-minute bucket: bucket ISO -> count (sparse). + site_visits: dict[str, int] = {} + + +def _bucket(now: datetime) -> str: + """Start of the 5-minute interval containing ``now``, as ISO string.""" + return now.replace(minute=now.minute // 5 * 5, second=0, microsecond=0).isoformat() + + +def _origin(url: str) -> str | None: + """The origin part of an https URL (scheme://host[:port]), else None.""" + try: + parsed = urlparse(url) + except ValueError: + return None + if parsed.scheme != "https" or not parsed.netloc: + return None + return f"https://{parsed.netloc}" + + +_SEGMENT = re.compile(r"[a-z0-9][a-z0-9_-]*") + + +def _internal_path(to: str) -> str | None: + """A valid internal page path ("/" or slug segments), else None.""" + path = to.split("?")[0].split("#")[0].strip("/") + if not path: + return "/" + if all(_SEGMENT.fullmatch(seg) for seg in path.split("/")): + return f"/{path}" + return None + + +class Store: + """In-memory analytics data plus the (IP, UA) -> visit session map.""" + + def __init__(self, path: Path) -> None: + self.path = path + self.data = Analytics() + if path.exists(): + try: + self.data = msgspec.json.decode(path.read_bytes(), type=Analytics) + except (msgspec.DecodeError, OSError): + pass # corrupt/unreadable file: start fresh + #: (ip, user-agent) -> index of the current visit in data.visits + self.sessions: dict[tuple[str, str], int] = {} + #: ip -> external https origin of the latest document GET carrying + #: one, stashed for the visit the client's initial ping starts. + #: Internal or absent referers never touch the table. + self.pending_referers: dict[str, str] = {} + + def _save(self) -> None: + """Rewrite the JSON file atomically (temp file + rename).""" + try: + fd, tmp = tempfile.mkstemp( + dir=self.path.parent, prefix=self.path.name, suffix=".tmp" + ) + with os.fdopen(fd, "wb") as f: + f.write(msgspec.json.encode(self.data)) + os.replace(tmp, self.path) + except OSError: + pass # analytics must never break page serving + + def _count(self, table: dict[str, int], key: str) -> None: + table[key] = table.get(key, 0) + 1 + + def _new_visit(self, entry: str, referer: str, key: tuple[str, str]) -> Visit: + now = datetime.now(UTC) + visit = Visit(start=now, entry=entry, referer=referer) + self.data.visits.append(visit) + self.sessions[key] = len(self.data.visits) - 1 + self._count(self.data.site_visits, _bucket(now)) + self._count(self.data.views.setdefault(entry, {}), _bucket(now)) + self._count( + self.data.transitions.setdefault(referer or "(direct)", {}), entry + ) + return visit + + def entry_referer(self, referer: str, own_origin: str, ip: str) -> None: + """Stash the entry referer of a document GET for ping attribution. + + Nothing is counted here — the client's initial /_a ping starts the + visit (only non-admin clients ping). Only a cross-origin https + referer updates the table; an internal or absent referer leaves any + stashed origin untouched. + """ + if not referer: + return + origin = _origin(referer) + if origin is None or origin == own_origin: + return + self.pending_referers[ip] = origin + + def ping(self, from_: str, to: str, ip: str, ua: str) -> None: + """Record a client navigation ping ({from, to} from pagerite.js). + + ``to`` is an internal path ("/...") or an https origin for exit + links; anything else is ignored. The transition is always counted; + the trail only grows on first sight of a page within the visit. + A ping with no known session starts a fresh visit, consuming the + referer stashed by the document GET if there is one. + """ + if to.startswith("/") and not to.startswith("//"): + target = _internal_path(to) or "" + else: + target = _origin(to) or "" + if not target or (not to.startswith("/") and target != to): + return + key = (ip, ua) + index = self.sessions.get(key) + fr = (_internal_path(from_) or "(direct)") if from_ else "(direct)" + if index is None or index >= len(self.data.visits): + # No known session: the initial ping of a fresh page load (or + # missing data after a server restart) — start a visit. + visit = self._new_visit(target, self.pending_referers.pop(ip, ""), key) + else: + visit = self.data.visits[index] + now = datetime.now(UTC) + if target.startswith("/"): + self._count(self.data.views.setdefault(target, {}), _bucket(now)) + self._count(self.data.transitions.setdefault(fr, {}), target) + # First-seen only: repeat pages and repeated exits don't append. + if visit.entry != target and target not in visit.trail: + visit.trail.append(target) + self._save() diff --git a/pagerite/app.py b/pagerite/app.py index a9d0168..2cbdb8d 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -20,15 +20,17 @@ from contextlib import asynccontextmanager from datetime import UTC, datetime from email.utils import format_datetime from pathlib import Path +from urllib.parse import urlparse import blake3 +import msgspec from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.responses import HTMLResponse, RedirectResponse, Response from fastapi_vue import Frontend from kanta import Kanta from pydantic import BaseModel -from pagerite import seed, views +from pagerite import analytics, seed, views from pagerite.__main__ import DEVMODE from pagerite.data import ( Data, @@ -43,6 +45,12 @@ from pagerite.markdown import has_h1, render, toggle_task DB_PATH = os.getenv("PAGERITE_DB", "pagerite.kantadb") +# Visit analytics go to their own JSON file, not the kanta database. +ANALYTICS_PATH = Path( + os.getenv("PAGERITE_ANALYTICS", DB_PATH.replace(".kantadb", "") + ".analytics.json") +) +analytics_store = analytics.Store(ANALYTICS_PATH) + # Our own data root; kanta edits it in place, reads are plain attribute access. data = Data() kanta = Kanta(DB_PATH, data) @@ -495,6 +503,41 @@ async def delete_page(path: str) -> None: _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +def _client_ip(request: Request) -> str: + """Client IP: first X-Forwarded-For hop (we sit behind a proxy), else + the direct peer.""" + forwarded = request.headers.get("x-forwarded-for", "").split(",")[0].strip() + return forwarded or (request.client.host if request.client else "") + + +class AnalyticsPing(BaseModel): + """Navigation ping from pagerite.js (see docs/analytics.md).""" + + fr: str = "" + to: str + + +@app.post("/_a", status_code=204) +async def analytics_ping(ping: AnalyticsPing, request: Request) -> None: + """Record a navigation ping ({fr, to}); fire-and-forget, never fails.""" + analytics_store.ping( + ping.fr, ping.to, _client_ip(request), + request.headers.get("user-agent", ""), + ) + + +def _track_entry(path: str, request: Request) -> None: + """Stash the referer of the document GET for the initial ping. + + Nothing is counted on the GET itself — the client's /_a ping starts the + visit, so bots and admin browsing never register. + """ + own_origin = f"https://{urlparse(str(request.base_url)).netloc}" + analytics_store.entry_referer( + request.headers.get("referer", ""), own_origin, _client_ip(request) + ) + + def _http_date(dt: datetime) -> str: """RFC 7231 date for the Last-Modified header.""" return format_datetime(dt.astimezone(UTC), usegmt=True) @@ -519,6 +562,18 @@ def _check_reserved(path: str) -> None: ) +@app.get("/_api/analytics") +async def get_analytics() -> Response: + """The collected visit analytics as JSON (see docs/analytics.md). + + Admin-only via the /_api forward-auth gate, like every management + endpoint. Powers the full-screen analytics viewer in the frontend. + """ + return Response( + msgspec.json.encode(analytics_store.data), media_type="application/json" + ) + + @app.websocket("/_api/ws/editor") async def editor_ws(ws: WebSocket) -> None: """Editor session: open pages, render previews, save — over one socket. @@ -699,6 +754,7 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response: etag = f'"{path}@{node.modified.timestamp()}v{data.version}"' if request.headers.get("if-none-match") == etag: return Response(status_code=304) + _track_entry(path, request) return HTMLResponse( views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, str(request.base_url).rstrip("/")), headers={ @@ -710,6 +766,7 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response: if node is not None and node.published and node.content is None: # Category label without a landing page: placeholder with the pen # to create it (404 — no page here, but the node is real). + _track_entry(path, request) return HTMLResponse( views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html), 404, @@ -724,4 +781,5 @@ async def show_page(request: Request, path: str) -> HTMLResponse | Response: for slug, item in sorted_nodes(data.menu): if item.published: return RedirectResponse(f"/{slug}") + _track_entry(path, request) return HTMLResponse(views.render_not_found(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html), 404)