Extended analytics data collection.
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
*.lock
|
||||
*.kantadb
|
||||
pagerite.analytics.json
|
||||
dbip-*.mmdb*
|
||||
/pagerite/frontend-build
|
||||
package-lock.json
|
||||
|
||||
|
||||
+27
-5
@@ -24,9 +24,9 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with
|
||||
nothing, so bots and admin browsing never register. Reloads are not
|
||||
visits: the ping is skipped (PerformanceNavigationTiming `reload`), so a
|
||||
refresh neither counts a second view nor logs a self-transition. The GET
|
||||
handler only stashes a cross-origin https `Referer` (origin part only) in
|
||||
an in-memory IP → referer table, consumed by the ping that starts the
|
||||
visit; internal or absent referers never touch the table.
|
||||
handler stashes a cross-origin https `Referer` (origin part only) and any
|
||||
`utm_*` query parameters in in-memory IP tables, consumed by the ping that
|
||||
starts the visit; internal or absent referers never touch the referer table.
|
||||
- **Internal fetch-navigations**: `to` is the target path, sent only after
|
||||
the swap actually happened (a failed swap falls back to a full load,
|
||||
whose initial ping counts the view instead — no gap, no double count).
|
||||
@@ -41,6 +41,19 @@ The client (`pagerite.js`) POSTs fire-and-forget pings to `/_a` with
|
||||
- The server validates `to`: internal paths must be valid slug paths
|
||||
("/" or `[a-z0-9_-]` segments), external ones are re-derived to the
|
||||
https origin and accepted only when the client sent exactly that.
|
||||
- The initial ping also records the visitor's `User-Agent` and
|
||||
`Accept-Language` headers. The first `Accept-Language` tag is stored as
|
||||
`lang` (e.g. `en-us`) and its region subtag, if present, is stored as
|
||||
an initial `country` (e.g. `US`).
|
||||
- The visitor IP is stored. A reverse-DNS lookup is attempted for each new
|
||||
visit and the result, when available, is cached in RAM and stored as
|
||||
`host`; local/reserved/multicast addresses are skipped.
|
||||
- If a DB-IP MMDB file (`dbip-*.mmdb` or `dbip-*.mmdb.gz`) is present in the
|
||||
repository root, it is loaded at startup and used to look up a more accurate
|
||||
`country`. The MMDB lookup and the reverse-DNS lookup run in background
|
||||
tasks after the visit is stored, so the `/ _a` response is never delayed.
|
||||
The decompressed `dbip-*.mmdb` file is kept in the repository root and
|
||||
ignored by git.
|
||||
|
||||
## Visits and sessions
|
||||
|
||||
@@ -49,17 +62,26 @@ There are no cookies. A visit is tied together by the (IP, User-Agent) pair
|
||||
direct peer): the first ping from a pair starts a new visit, subsequent
|
||||
pings extend it. Pings arriving with no known session (server restart)
|
||||
start a fresh visit from the first ping — treated as missing data rather
|
||||
than dropped. The (IP, UA) → visit map and the IP → entry-referer table
|
||||
are in-memory only; IPs are never persisted.
|
||||
than dropped. The (IP, UA) → visit map and the IP → entry-referer/UTM
|
||||
tables are in-memory only, but the IP and any resolvable reverse-DNS host
|
||||
name are stored on the `Visit` record itself.
|
||||
|
||||
Each `Visit` record:
|
||||
|
||||
- `start` — timestamp of the first event,
|
||||
- `entry` — first page (path) seen,
|
||||
- `referer` — external https origin of the initial load, `""` for direct,
|
||||
- `ip` — visitor IP address (first `X-Forwarded-For` hop, or direct peer),
|
||||
- `host` — reverse-DNS host name for `ip` when resolvable, else `""`,
|
||||
- `trail` — everything seen afterwards in first-seen order: page paths and
|
||||
external exit origins. Re-visiting an already seen page (incl. the entry)
|
||||
does not append.
|
||||
- `lang` — first `Accept-Language` tag, lowercased (e.g. `en-us`),
|
||||
- `country` — two-letter country code. Initially derived from the
|
||||
`Accept-Language` region subtag, but overwritten by the DB-IP MMDB result
|
||||
when a database is available,
|
||||
- `ua` — raw `User-Agent` string from the initial ping,
|
||||
- `utm` — `utm_*` query parameters from the landing URL, as a dict.
|
||||
|
||||
## Aggregates
|
||||
|
||||
|
||||
+138
-30
@@ -8,7 +8,13 @@
|
||||
// See docs/analytics.md for the data format.
|
||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { RANGES } from './analytics/time.js'
|
||||
import { calcTotalViews, formatRecentVisits } from './analytics/format.js'
|
||||
import {
|
||||
calcTotalViews,
|
||||
countByField,
|
||||
countUtmTags,
|
||||
formatCounts,
|
||||
formatVisitRows,
|
||||
} from './analytics/format.js'
|
||||
import TransitionGraph from './TransitionGraph.vue'
|
||||
import VisitorCharts from './VisitorCharts.vue'
|
||||
|
||||
@@ -55,7 +61,15 @@ watch(range, (r) => {
|
||||
}
|
||||
})
|
||||
|
||||
const recentVisits = computed(() => formatRecentVisits(visits.value, pageTree.value))
|
||||
const visitRows = computed(() => formatVisitRows(visits.value, pageTree.value))
|
||||
const languageCounts = computed(() => countByField(visits.value, 'lang'))
|
||||
const countryCounts = computed(() => countByField(visits.value, 'country'))
|
||||
const utmTagCounts = computed(() => countUtmTags(visits.value))
|
||||
const hasBreakdowns = computed(() =>
|
||||
languageCounts.value.length > 0
|
||||
|| countryCounts.value.length > 0
|
||||
|| utmTagCounts.value.length > 0,
|
||||
)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -79,22 +93,62 @@ const recentVisits = computed(() => formatRecentVisits(visits.value, pageTree.va
|
||||
<div><strong>{{ totalViews }}</strong> page views</div>
|
||||
</section>
|
||||
|
||||
<section v-if="hasBreakdowns" class="breakdowns">
|
||||
<div v-if="languageCounts.length" class="breakdown">
|
||||
<h3>Languages</h3>
|
||||
<p>{{ formatCounts(languageCounts) }}</p>
|
||||
</div>
|
||||
<div v-if="countryCounts.length" class="breakdown">
|
||||
<h3>Countries</h3>
|
||||
<p>{{ formatCounts(countryCounts) }}</p>
|
||||
<p class="note">from Accept-Language or DB-IP when available</p>
|
||||
</div>
|
||||
<div v-if="utmTagCounts.length" class="breakdown">
|
||||
<h3>UTM tags</h3>
|
||||
<p>{{ formatCounts(utmTagCounts) }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<VisitorCharts :data="data" :range="range" />
|
||||
<TransitionGraph :data="data" :range="range" :page-tree="pageTree" @close="emit('close')" />
|
||||
|
||||
<section>
|
||||
<h2>Recent visits</h2>
|
||||
<ul v-if="recentVisits.length" class="visits">
|
||||
<li v-for="(v, i) in recentVisits" :key="i">
|
||||
<span class="when">{{ v.when }}</span>
|
||||
<span class="trail">
|
||||
<a v-for="(s, si) in v.steps" :key="si"
|
||||
<div v-if="visitRows.length" class="visit-table-wrap">
|
||||
<table class="visit-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>when</th>
|
||||
<th>trail</th>
|
||||
<th>referer</th>
|
||||
<th>ip</th>
|
||||
<th>host</th>
|
||||
<th>lang</th>
|
||||
<th>country</th>
|
||||
<th>ua</th>
|
||||
<th>utm</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="(v, i) in visitRows" :key="i">
|
||||
<td class="when">{{ v.when }}</td>
|
||||
<td class="trail">
|
||||
<a v-for="(s, si) in v.trail" :key="si"
|
||||
:href="s.path" :title="s.title" @click="emit('close')">
|
||||
{{ s.slug }}
|
||||
</a>
|
||||
</span>
|
||||
</li>
|
||||
</ul>
|
||||
</td>
|
||||
<td>{{ v.referer }}</td>
|
||||
<td>{{ v.ip }}</td>
|
||||
<td>{{ v.host }}</td>
|
||||
<td>{{ v.lang }}</td>
|
||||
<td>{{ v.country }}</td>
|
||||
<td class="ua" :title="v.ua">{{ v.ua }}</td>
|
||||
<td>{{ v.utm }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p v-else class="empty">no visits recorded yet</p>
|
||||
</section>
|
||||
</template>
|
||||
@@ -177,36 +231,90 @@ const recentVisits = computed(() => formatRecentVisits(visits.value, pageTree.va
|
||||
}
|
||||
.totals strong { font-size: 1.5rem; }
|
||||
|
||||
.visits {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
.visit-table-wrap {
|
||||
overflow-x: auto;
|
||||
}
|
||||
.visits li {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
padding: 0.2rem 0;
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
.visits .when {
|
||||
flex-shrink: 0;
|
||||
color: var(--muted);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.visits .trail {
|
||||
|
||||
.visit-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-family: monospace;
|
||||
word-break: normal;
|
||||
font-size: 0.82rem;
|
||||
line-height: 1.3;
|
||||
}
|
||||
|
||||
.visit-table th,
|
||||
.visit-table td {
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-bottom: 1px solid var(--line);
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
.visit-table th {
|
||||
color: var(--muted);
|
||||
font-weight: normal;
|
||||
text-transform: lowercase;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: var(--bg, Canvas);
|
||||
}
|
||||
|
||||
.visit-table .when {
|
||||
white-space: nowrap;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.visit-table .trail {
|
||||
max-width: 20rem;
|
||||
overflow-wrap: break-word;
|
||||
}
|
||||
.visits .trail a {
|
||||
|
||||
.visit-table .trail a {
|
||||
color: var(--text);
|
||||
text-decoration: none;
|
||||
}
|
||||
.visits .trail a:hover { color: var(--accent); }
|
||||
.visits .trail a + a {
|
||||
|
||||
.visit-table .trail a:hover { color: var(--accent); }
|
||||
|
||||
.visit-table .trail a + a {
|
||||
margin-left: 0.5rem;
|
||||
}
|
||||
|
||||
.visit-table .ua {
|
||||
max-width: 18rem;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.breakdowns {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr));
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.breakdown {
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.breakdown h3 {
|
||||
margin: 0 0 0.3rem;
|
||||
font-size: 0.9rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.breakdown p {
|
||||
margin: 0;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.breakdown .note {
|
||||
font-size: 0.8rem;
|
||||
color: var(--muted);
|
||||
margin-top: 0.2rem;
|
||||
}
|
||||
|
||||
.empty, .loading, .error { color: var(--muted); }
|
||||
.error { color: var(--error, #c00); }
|
||||
</style>
|
||||
|
||||
@@ -52,3 +52,70 @@ export function formatRecentVisits(visits, pageTree, limit = 50) {
|
||||
.filter((v) => v.steps.length)
|
||||
.slice(0, limit)
|
||||
}
|
||||
|
||||
/**
|
||||
* Count distinct values of a visit field, sorted most-common first.
|
||||
* Returns an array of [value, count] pairs.
|
||||
*/
|
||||
export function countByField(visits, field) {
|
||||
const counts = {}
|
||||
for (const v of visits || []) {
|
||||
const value = v[field]
|
||||
if (!value) continue
|
||||
counts[value] = (counts[value] || 0) + 1
|
||||
}
|
||||
return Object.entries(counts).sort((a, b) => b[1] - a[1])
|
||||
}
|
||||
|
||||
/**
|
||||
* Count UTM parameter occurrences across visits. Each distinct
|
||||
* ``parameter: value`` pair is counted separately. Returns [pair, count].
|
||||
*/
|
||||
export function countUtmTags(visits) {
|
||||
const counts = {}
|
||||
for (const v of visits || []) {
|
||||
for (const [key, value] of Object.entries(v.utm || {})) {
|
||||
const label = `${key}: ${value}`
|
||||
counts[label] = (counts[label] || 0) + 1
|
||||
}
|
||||
}
|
||||
return Object.entries(counts).sort((a, b) => b[1] - a[1])
|
||||
}
|
||||
|
||||
/** Format a list of [value, count] pairs for inline display. */
|
||||
export function formatCounts(entries) {
|
||||
return entries.map(([value, count]) => `${value} (${count})`).join(', ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Format raw visit records as rows for a technical table. Returns objects
|
||||
* with display strings; missing values become "—". ``trail`` joins page
|
||||
* titles (when known) with " -> ".
|
||||
*/
|
||||
export function formatVisitRows(visits, pageTree) {
|
||||
const titles = buildTitleMap(pageTree)
|
||||
return [...(visits || [])].reverse().map((v) => {
|
||||
const trail = [v.entry, ...(v.trail || [])]
|
||||
.filter((p) => p?.startsWith('/'))
|
||||
.map((p) => ({
|
||||
path: p,
|
||||
slug: slugOf(p),
|
||||
title: titles.get(p) || '',
|
||||
}))
|
||||
const utm = Object.entries(v.utm || {})
|
||||
.map(([k, value]) => `${k}=${value}`)
|
||||
.join(', ')
|
||||
const dash = (s) => (s || '—')
|
||||
return {
|
||||
when: new Date(v.start).toLocaleString(),
|
||||
trail,
|
||||
referer: dash(v.referer),
|
||||
ip: dash(v.ip),
|
||||
host: dash(v.host),
|
||||
lang: dash(v.lang),
|
||||
country: dash(v.country),
|
||||
ua: dash(v.ua),
|
||||
utm: utm || '—',
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+135
-18
@@ -3,10 +3,12 @@
|
||||
Events come from navigation pings POSTed to /_a by pagerite.js: the first
|
||||
ping on page load starts a visit, later pings extend it, and pings with no
|
||||
known session start a fresh one (missing data, not dropped). The document
|
||||
GET handler only stashes the entry referer (external https origin) in an
|
||||
in-memory IP -> referer table, consumed when the ping starts the visit;
|
||||
nothing is counted without a ping (bots and admin browsing stay invisible).
|
||||
The session map is in-memory only; IPs are never persisted.
|
||||
GET handler stashes the entry referer (external https origin) and any
|
||||
utm_* query parameters in in-memory IP tables, consumed when the ping
|
||||
starts the visit; nothing is counted without a ping (bots and admin
|
||||
browsing stay invisible). The session map is in-memory only. The visitor
|
||||
IP and, when available, its reverse-DNS host name are stored on the visit
|
||||
record itself.
|
||||
|
||||
Data is a msgspec Struct JSON-dumped to its own file (not the kanta db),
|
||||
rewritten atomically on every recorded event.
|
||||
@@ -17,7 +19,7 @@ import re
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -34,7 +36,19 @@ class Visit(msgspec.Struct, omit_defaults=True):
|
||||
entry: str
|
||||
#: External https origin of the initial load, "" for direct visits.
|
||||
referer: str = ""
|
||||
#: Visitor IP address (first X-Forwarded-For hop or direct peer).
|
||||
ip: str = ""
|
||||
#: Reverse-DNS host name for ``ip`` when resolvable, else "".
|
||||
host: str = ""
|
||||
trail: list[str] = []
|
||||
#: First Accept-Language tag, lowercased (e.g. "en-us").
|
||||
lang: str = ""
|
||||
#: Two-letter region subtag derived from ``lang`` (e.g. "US"), or "".
|
||||
country: str = ""
|
||||
#: Raw User-Agent header from the initial ping.
|
||||
ua: str = ""
|
||||
#: UTM query parameters from the landing URL, keyed by parameter name.
|
||||
utm: dict[str, str] = {}
|
||||
|
||||
|
||||
class Analytics(msgspec.Struct, omit_defaults=True):
|
||||
@@ -80,6 +94,37 @@ def _internal_path(to: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_accept_language(value: str) -> tuple[str, str]:
|
||||
"""First Accept-Language tag and the region/country subtag if present.
|
||||
|
||||
``en-US, fr;q=0.9`` -> ("en-us", "US"). Wildcards and missing regions
|
||||
produce an empty country. The region is intentionally approximate:
|
||||
it reflects the browser's language preference, not geo-location.
|
||||
"""
|
||||
if not value:
|
||||
return "", ""
|
||||
tag = value.split(",")[0].split(";")[0].strip()
|
||||
if not tag or tag == "*":
|
||||
return "", ""
|
||||
lang = tag.lower()
|
||||
country = ""
|
||||
# Region subtags follow the initial language tag (en-US, zh-Hans-CN).
|
||||
# A bare two-letter tag such as "fr" is a language code, not a region.
|
||||
for part in reversed(tag.split("-")[1:]):
|
||||
if len(part) == 2 and part.isalpha():
|
||||
country = part.upper()
|
||||
break
|
||||
return lang, country
|
||||
|
||||
|
||||
def _utm_tags(query: str) -> dict[str, str]:
|
||||
"""UTM parameters from a query string, keeping only the first value."""
|
||||
if not query:
|
||||
return {}
|
||||
parsed = parse_qs(query, keep_blank_values=True)
|
||||
return {k: v[0] for k, v in parsed.items() if k.startswith("utm_")}
|
||||
|
||||
|
||||
class Store:
|
||||
"""In-memory analytics data plus the (IP, UA) -> visit session map."""
|
||||
|
||||
@@ -97,6 +142,11 @@ class Store:
|
||||
#: one, stashed for the visit the client's initial ping starts.
|
||||
#: Internal or absent referers never touch the table.
|
||||
self.pending_referers: dict[str, str] = {}
|
||||
#: ip -> utm_* query parameters from the latest document GET that
|
||||
#: carried any, stashed for the visit the client's initial ping starts.
|
||||
#: Only non-empty sets are stored, so a later parameter-less page
|
||||
#: does not overwrite an earlier tagged landing URL.
|
||||
self.pending_utms: dict[str, dict[str, str]] = {}
|
||||
|
||||
def _save(self) -> None:
|
||||
"""Rewrite the JSON file atomically (temp file + rename)."""
|
||||
@@ -113,9 +163,28 @@ class Store:
|
||||
def _count(self, table: dict[str, int], key: str) -> None:
|
||||
table[key] = table.get(key, 0) + 1
|
||||
|
||||
def _new_visit(self, entry: str, referer: str, key: tuple[str, str]) -> Visit:
|
||||
def _new_visit(
|
||||
self,
|
||||
entry: str,
|
||||
referer: str,
|
||||
key: tuple[str, str],
|
||||
ip: str = "",
|
||||
lang: str = "",
|
||||
country: str = "",
|
||||
ua: str = "",
|
||||
utm: dict[str, str] | None = None,
|
||||
) -> Visit:
|
||||
now = datetime.now(UTC)
|
||||
visit = Visit(start=now, entry=entry, referer=referer)
|
||||
visit = Visit(
|
||||
start=now,
|
||||
entry=entry,
|
||||
referer=referer,
|
||||
ip=ip,
|
||||
lang=lang,
|
||||
country=country,
|
||||
ua=ua,
|
||||
utm=utm or {},
|
||||
)
|
||||
self.data.visits.append(visit)
|
||||
self.sessions[key] = len(self.data.visits) - 1
|
||||
self._count(self.data.site_visits, _bucket(now))
|
||||
@@ -125,43 +194,90 @@ class Store:
|
||||
)
|
||||
return visit
|
||||
|
||||
def entry_referer(self, referer: str, own_origin: str, ip: str) -> None:
|
||||
"""Stash the entry referer of a document GET for ping attribution.
|
||||
def enrich_visit(
|
||||
self,
|
||||
index: int,
|
||||
*,
|
||||
host: str = "",
|
||||
country: str = "",
|
||||
) -> None:
|
||||
"""Fill in host/geoip fields on an existing visit after async lookups."""
|
||||
if index < 0 or index >= len(self.data.visits):
|
||||
return
|
||||
visit = self.data.visits[index]
|
||||
changed = False
|
||||
if host and not visit.host:
|
||||
visit.host = host
|
||||
changed = True
|
||||
if country:
|
||||
visit.country = country
|
||||
changed = True
|
||||
if changed:
|
||||
self._save()
|
||||
|
||||
def entry_referer(
|
||||
self, referer: str, own_origin: str, ip: str, query: str = ""
|
||||
) -> None:
|
||||
"""Stash the entry referer and UTM tags of a document GET for ping attribution.
|
||||
|
||||
Nothing is counted here — the client's initial /_a ping starts the
|
||||
visit (only non-admin clients ping). Only a cross-origin https
|
||||
referer updates the table; an internal or absent referer leaves any
|
||||
stashed origin untouched.
|
||||
stashed origin untouched. UTM parameters are kept only when the
|
||||
landing URL actually carries them, so a subsequent parameter-less page
|
||||
does not erase an earlier tagged landing.
|
||||
"""
|
||||
if not referer:
|
||||
return
|
||||
if referer:
|
||||
origin = _origin(referer)
|
||||
if origin is None or origin == own_origin:
|
||||
return
|
||||
if origin is not None and origin != own_origin:
|
||||
self.pending_referers[ip] = origin
|
||||
utms = _utm_tags(query)
|
||||
if utms:
|
||||
self.pending_utms[ip] = utms
|
||||
|
||||
def ping(self, from_: str, to: str, ip: str, ua: str) -> None:
|
||||
def ping(
|
||||
self,
|
||||
from_: str,
|
||||
to: str,
|
||||
ip: str,
|
||||
ua: str,
|
||||
accept_language: str = "",
|
||||
) -> int | None:
|
||||
"""Record a client navigation ping ({from, to} from pagerite.js).
|
||||
|
||||
``to`` is an internal path ("/...") or an https origin for exit
|
||||
links; anything else is ignored. The transition is always counted;
|
||||
the trail only grows on first sight of a page within the visit.
|
||||
A ping with no known session starts a fresh visit, consuming the
|
||||
referer stashed by the document GET if there is one.
|
||||
referer and UTM tags stashed by the document GET if there are any.
|
||||
|
||||
Returns the index of the new visit when one is created, so callers
|
||||
can enrich it later with non-blocking lookups (host, geoip country).
|
||||
"""
|
||||
if to.startswith("/") and not to.startswith("//"):
|
||||
target = _internal_path(to) or ""
|
||||
else:
|
||||
target = _origin(to) or ""
|
||||
if not target or (not to.startswith("/") and target != to):
|
||||
return
|
||||
return None
|
||||
key = (ip, ua)
|
||||
index = self.sessions.get(key)
|
||||
fr = (_internal_path(from_) or "(direct)") if from_ else "(direct)"
|
||||
if index is None or index >= len(self.data.visits):
|
||||
# No known session: the initial ping of a fresh page load (or
|
||||
# missing data after a server restart) — start a visit.
|
||||
visit = self._new_visit(target, self.pending_referers.pop(ip, ""), key)
|
||||
lang, country = _parse_accept_language(accept_language)
|
||||
index = len(self.data.visits)
|
||||
self._new_visit(
|
||||
target,
|
||||
self.pending_referers.pop(ip, ""),
|
||||
key,
|
||||
ip=ip,
|
||||
lang=lang,
|
||||
country=country,
|
||||
ua=ua,
|
||||
utm=self.pending_utms.pop(ip, {}),
|
||||
)
|
||||
else:
|
||||
visit = self.data.visits[index]
|
||||
now = datetime.now(UTC)
|
||||
@@ -172,3 +288,4 @@ class Store:
|
||||
if visit.entry != target and target not in visit.trail:
|
||||
visit.trail.append(target)
|
||||
self._save()
|
||||
return index if index is not None and index < len(self.data.visits) else None
|
||||
|
||||
+133
-6
@@ -12,13 +12,19 @@ walking the tree (``resolve``), moves are slot detach/attach
|
||||
(``find_slot``) with a fresh order key from the new siblings.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import ipaddress
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import format_datetime
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -51,6 +57,74 @@ ANALYTICS_PATH = Path(
|
||||
)
|
||||
analytics_store = analytics.Store(ANALYTICS_PATH)
|
||||
|
||||
|
||||
# Repository root from this file's location (pagerite/app.py -> ..).
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _geoip_db_path() -> Path | None:
|
||||
"""Find a DB-IP MMDB in the repo root, preferring an already-decompressed
|
||||
``.mmdb`` over the matching ``.mmdb.gz``. Returns None if none is present.
|
||||
"""
|
||||
mmdb = sorted(_REPO_ROOT.glob("dbip-*.mmdb"))
|
||||
if mmdb:
|
||||
return mmdb[0]
|
||||
gz = sorted(_REPO_ROOT.glob("dbip-*.mmdb.gz"))
|
||||
if gz:
|
||||
return gz[0]
|
||||
return None
|
||||
|
||||
|
||||
class GeoIP:
|
||||
"""Lazy DB-IP MMDB reader. Call ``_load()`` once at startup before
|
||||
concurrent requests arrive; ``country()`` is read-only and safe to call
|
||||
from ``asyncio.to_thread`` workers afterwards.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._reader: object | None = None
|
||||
|
||||
def _decompress(self, source: Path, target: Path) -> None:
|
||||
if target.exists():
|
||||
return
|
||||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||
with gzip.open(source, "rb") as src, open(tmp, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
os.replace(tmp, target)
|
||||
|
||||
def _load(self) -> None:
|
||||
if self._reader is not None:
|
||||
return
|
||||
source = _geoip_db_path()
|
||||
if source is None:
|
||||
return
|
||||
if source.suffix == ".gz":
|
||||
target = source.with_suffix("")
|
||||
self._decompress(source, target)
|
||||
source = target
|
||||
try:
|
||||
import maxminddb
|
||||
|
||||
self._reader = maxminddb.open_database(str(source))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def country(self, ip: str) -> str:
|
||||
"""Two-letter ISO country code for ``ip``, or "" when unavailable."""
|
||||
if not ip or self._reader is None:
|
||||
return ""
|
||||
try:
|
||||
rec = self._reader.get(ip)
|
||||
if rec:
|
||||
return (rec.get("country") or {}).get("iso_code", "")
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
_geoip = GeoIP()
|
||||
|
||||
|
||||
# Our own data root; kanta edits it in place, reads are plain attribute access.
|
||||
data = Data()
|
||||
kanta = Kanta(DB_PATH, data)
|
||||
@@ -150,10 +224,13 @@ def _seed(data: Data) -> None:
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||
"""Open the database, migrate legacy content, load assets."""
|
||||
"""Open the database, migrate legacy content, load assets, load GeoIP."""
|
||||
await kanta.open()
|
||||
_migrate_legacy()
|
||||
await frontend.load()
|
||||
# Decompress/open the DB-IP MMDB once at startup. Lookups are then
|
||||
# read-only and safe to run in background ``to_thread`` workers.
|
||||
await asyncio.to_thread(_geoip._load)
|
||||
yield
|
||||
await kanta.close()
|
||||
|
||||
@@ -510,6 +587,43 @@ def _client_ip(request: Request) -> str:
|
||||
return forwarded or (request.client.host if request.client else "")
|
||||
|
||||
|
||||
@lru_cache(maxsize=4096)
|
||||
def _cached_ptr(ip: str) -> str:
|
||||
"""Reverse-DNS lookup with in-RAM LRU cache. Returns the host name or ""."""
|
||||
if not ip:
|
||||
return ""
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return ""
|
||||
if addr.is_private or addr.is_loopback or addr.is_reserved or addr.is_multicast or addr.is_link_local:
|
||||
return ""
|
||||
try:
|
||||
host, _, _ = socket.gethostbyaddr(ip)
|
||||
except socket.herror:
|
||||
return ""
|
||||
return host
|
||||
|
||||
|
||||
async def _lookup_host(ip: str) -> str:
|
||||
"""Async wrapper around ``_cached_ptr``; runs the blocking lookup in a thread."""
|
||||
return await asyncio.to_thread(_cached_ptr, ip)
|
||||
|
||||
|
||||
async def _geoip_country(ip: str) -> str:
|
||||
"""Async wrapper around the DB-IP MMDB lookup."""
|
||||
return await asyncio.to_thread(_geoip.country, ip)
|
||||
|
||||
|
||||
async def _enrich_visit(index: int, ip: str) -> None:
|
||||
"""Run non-blocking reverse-DNS and geoip enrichment for a new visit."""
|
||||
if not ip:
|
||||
return
|
||||
host = await _lookup_host(ip)
|
||||
country = await _geoip_country(ip)
|
||||
analytics_store.enrich_visit(index, host=host, country=country)
|
||||
|
||||
|
||||
class AnalyticsPing(BaseModel):
|
||||
"""Navigation ping from pagerite.js (see docs/analytics.md)."""
|
||||
|
||||
@@ -519,22 +633,35 @@ class AnalyticsPing(BaseModel):
|
||||
|
||||
@app.post("/_a", status_code=204)
|
||||
async def analytics_ping(ping: AnalyticsPing, request: Request) -> None:
|
||||
"""Record a navigation ping ({fr, to}); fire-and-forget, never fails."""
|
||||
analytics_store.ping(
|
||||
ping.fr, ping.to, _client_ip(request),
|
||||
"""Record a navigation ping ({fr, to}); fire-and-forget, never fails.
|
||||
|
||||
The reverse-DNS and DB-IP geoip lookups happen in a background task so
|
||||
the response is never delayed by slow DNS or the first MMDB decompress.
|
||||
"""
|
||||
ip = _client_ip(request)
|
||||
index = analytics_store.ping(
|
||||
ping.fr,
|
||||
ping.to,
|
||||
ip,
|
||||
request.headers.get("user-agent", ""),
|
||||
request.headers.get("accept-language", ""),
|
||||
)
|
||||
if index is not None:
|
||||
asyncio.create_task(_enrich_visit(index, ip))
|
||||
|
||||
|
||||
def _track_entry(path: str, request: Request) -> None:
|
||||
"""Stash the referer of the document GET for the initial ping.
|
||||
"""Stash the referer and UTM tags of the document GET for the initial ping.
|
||||
|
||||
Nothing is counted on the GET itself — the client's /_a ping starts the
|
||||
visit, so bots and admin browsing never register.
|
||||
"""
|
||||
own_origin = f"https://{urlparse(str(request.base_url)).netloc}"
|
||||
analytics_store.entry_referer(
|
||||
request.headers.get("referer", ""), own_origin, _client_ip(request)
|
||||
request.headers.get("referer", ""),
|
||||
own_origin,
|
||||
_client_ip(request),
|
||||
str(request.url.query),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ dependencies = [
|
||||
"html5tagger>=2.0.0",
|
||||
"kanta>=0.8.1",
|
||||
"markdown-it-py>=4.2.0",
|
||||
"maxminddb>=3.1.1",
|
||||
"mdit-py-plugins>=0.6.1",
|
||||
"pygments>=2.20.0",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user