From 218313ba7a9ced8aca38382aa9f8d2a6eb0afe34 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:14:10 +0000 Subject: [PATCH] Phase 1: content negotiation (?lang=, Accept-Language) + localization/migration design docs - pagerite/i18n.py: parse_accept_language, select_language, Translation struct, get_translation/available_languages stubs - app.py: language threaded through show_page/_render_html/_cached_body, ETag and Vary: accept-language - views.py: , translated nav titles with fallback, canonical with ?lang=, hreflang alternates - pagerite.js: strip ?lang= via replaceState, send it as Accept-Language on fetch-navigation/preloads, lang-aware page cache - docs/localization.md: negotiation + URL scheme + phase-2 fragment model - docs/migrate.md: migrate_v3 content-addressed chunk storage plan --- docs/localization.md | 219 +++++++++++++++++++++++++++++++++++++++ docs/migrate.md | 142 +++++++++++++++++++++++++ frontend/src/pagerite.js | 34 ++++-- pagerite/app.py | 42 +++++--- pagerite/i18n.py | 89 ++++++++++++++++ pagerite/views.py | 108 +++++++++++++------ 6 files changed, 584 insertions(+), 50 deletions(-) create mode 100644 docs/localization.md create mode 100644 docs/migrate.md create mode 100644 pagerite/i18n.py diff --git a/docs/localization.md b/docs/localization.md new file mode 100644 index 0000000..704336c --- /dev/null +++ b/docs/localization.md @@ -0,0 +1,219 @@ +# Localization + +Pages are served in the visitor's language based on a `?lang=` query +parameter or the `Accept-Language` header. + +- **Phase 1 (implemented):** negotiation, URL scheme, caching, rendering + plumbing. Translations are consumed through a stub interface; the database + still holds only the original language. +- **Phase 2 (final plan):** gettext-style fragment storage in the + database — machine-translated chunks plus user override patches, assembled + at render time. Storage details in `docs/migrate.md`. + +## Phase 1: negotiation and URLs + +### Language selection + +Deliberately simple — **q-values are ignored**: + +- All known `Accept-Language` implementations send the header **in order of + preference**, so we parse it as an ordered list and never reorder. +- Selection rule (`select_language` in `pagerite/i18n.py`): + 1. If `?lang=` is present, use it (if a translation exists; otherwise + fall through to header logic). + 2. If the article's original language appears anywhere in the header list, + use the **original**. Rationale: an AI translation is strictly worse + than the original for anyone who has that language configured at all + (e.g. `fi-FI, fi, en-US, en` gets English, not machine-translated + Finnish). + 3. Otherwise walk the header list in order and use the first language for + which a translation exists. + 4. Fall back to the original. + +Region tags normalize to their base subtag (`fi-FI` → `fi`). + +### URLs: pretty for users, indexable for search engines + +- Canonical URLs stay pretty (`/some-page`). Each language version is + addressable as `/some-page?lang=fi` so search engines can index them. +- `` points to the page **itself including the query** + (each language version is its own canonical). +- `` entries point to every other language + version (with `?lang=`), plus `x-default` for the plain URL. +- On page load, pagerite.js removes the `?lang=` query via + `history.replaceState` (restoring the pretty URL) but remembers the language + in a JS variable. All fetch-navigation and preloads it performs afterwards + send that language in the `Accept-Language` header, so the chosen language + sticks for the session of clicks. +- A full page refresh or a shared link resets to automatic selection (header + only). This gives a clean one-time override without cookies. + +### Response correctness + +- Content responses carry `Vary: accept-language` (added to the existing + `accept-encoding` vary). +- `_cached_body` and the page ETag include the **selected language** (not the + raw header, which would blow up the cache key space). +- `` reflects the served language. + +### Rendering + +- The translated Markdown goes through the same `markdown.render` pipeline. +- Navigation/sidebar titles come from the translation's title map, with + per-node fallback to the original title (a partially translated tree must + still render). +- Fixed UI strings ("Not Found" etc.) and the editor UI stay English for now. +- The markdown typographer (SmartyPants) is English-centric; per-language + typographer options are a possible follow-up, not blocking. + +## Phase 2: fragment-based translation storage (draft) + +Phase 1 assumed whole-page translated Markdown delivered from outside. The +refined model is gettext-style: an article has **one primary version** (its +`content`, in its own language) plus, per target language, **machine +fragments** (translated chunks of Markdown) and **user patches** (minimal +editor overrides). Both are stored in the database and assembled into the +served Markdown at render time. + +### The scenario this must handle + +1. Article written in English. +2. Machine-translated into Spanish → fragments stored. +3. Editor fixes one Spanish paragraph and changes a link elsewhere to point + at a Spanish resource → user patch hunks stored. +4. English article edited → the edited chunk's key changes; its Spanish + fragment no longer matches. +5. Page requested before the machine translation refreshes → served as a + **hybrid**: old fragments for unchanged chunks, plain English for the + edited chunk. User patches are attempted against this hybrid, best effort, + each hunk independently: the text fix is stale (its search text no longer + exists) and silently skipped; the link change still applies even though + the link sits in the now-English paragraph. +6. Machine translation refreshes → full Spanish again, with both patch hunks + applying. + +### Chunks + +`chunk_markdown(markdown)` splits the source into block-level chunks — +blank-line-separated blocks: headings, paragraphs, code fences (kept whole), +list blocks, tables, HTML blocks. A chunk's identity is its **source text**, +gettext-msgid style: + +```python +chunk_key = blake3(normalize(chunk_text), digest_size=16).hex() +``` + +(`normalize`: strip trailing whitespace per line, collapse surrounding blank +lines — so whitespace-only source edits don't invalidate translations.) + +Consequences: + +- Editing the English source invalidates exactly the edited chunks; all + other fragments keep applying. Stale fragments are simply never referenced + again and can be garbage-collected lazily (or left; they are tiny). +- No explicit "source version" bookkeeping is needed — staleness falls out + of the keys. + +### User patches + +Editors always edit **full Markdown** in the existing editor UX — never +fragments. When editing a translated view (`?lang=es`), the editor is loaded +with the *current hybrid Markdown*; on save, the server computes a minimal +diff against that hybrid and stores it as a patch: + +```python +class Patch(msgspec.Struct, omit_defaults=True): + """One editing session's overrides, applied independently per hunk.""" + + hunks: list[tuple[str, str]] = [] # (search, replace) on hybrid Markdown +``` + +Hunks are produced from `difflib.SequenceMatcher` on the hybrid vs. the +edited text at block granularity: each `replace`/`delete`/`insert` opcode +becomes one `(search, replace)` pair, with the preceding block's tail as +left context for `insert` (pure inserts have empty search context otherwise). +Application is dead simple: + +```python +def apply_patch(hybrid: str, patch: Patch) -> str: + for search, replace in patch.hunks: + if search and search in hybrid: + hybrid = hybrid.replace(search, replace, 1) + # missing search text = stale hunk -> silently skipped + return hybrid +``` + +Per-hunk independence is the robustness property from the scenario: a stale +text fix does not block a still-valid link change. Patches are stored as an +ordered list and applied in order. + +### Storage + +Full storage design and the `migrate_v3` restructuring live in +`docs/migrate.md`. The short version, as it concerns this document: + +- Originals **and** translations are content-addressed text chunks in flat + stores: `Data.chunks: dict[hash, str]` and + `Data.trans: dict[f"{chunk_hash}:{lang}", str]` — path-independent, so + repeated paragraphs and menu titles are translated once and article moves + touch nothing. `Node.chunks: list[hash]` gives each article its order. +- `Node` gains **`language: str = ""`**, inherited down the tree like + `banner` (empty = nearest ancestor, front page last, site default `en` + final). `select_language` and `` use the resolved value instead + of the global `ORIGINAL_LANGUAGE` constant. + - **Known weakness:** changing a page's (or subtree's) `language` after + translations exist mis-keys everything — translations are keyed by + *source* chunks, so old entries silently stop matching and user patches + (searching for old-hybrid text) mostly go stale. That is acceptable: + the orphaned data is harmless and translations regenerate. We do not + migrate translations across a language change. +- Article paths are stored and keyed **without leading slashes** + (`"docs/setup"`, front page `""`); slashes are added only in hrefs. + +### Render pipeline (replaces the phase-1 `get_translation` stub) + +```python +def get_translation(path, lang, data) -> Translation | None: + if lang not in node.langs: + return None + hybrid = "\n\n".join( + chunks[h] if h in node.no_trans else trans.get(f"{h}:{lang}", chunks[h]) + for h in node.chunks + ) + for patch in data.patches.get(f"{path}:{lang}", []): + hybrid = apply_patch(hybrid, patch) + return Translation(markdown=hybrid, titles=title_map(data, lang)) +``` + +- Availability is an article-level index: `node.langs: dict[lang, True]`, + maintained by the translation writers (translator job, patch saves) in the + same transaction as their data writes — rendering, language selection and + hreflang never probe the `trans` store chunk by chunk. A stale key is + benign (the "translation" just renders as the original). +- `titles` for nav/sidebar/cards: each node's translated title is + `trans.get(f"{hash(node.title)}:{lang}")` with per-node fallback — one dict + lookup per nav item at render time. +- Cache invalidation: writes to `chunks` / `trans` / `patches` (translator, + editor saves) call `_invalidate_pages()`, same as content writes. + +### Editor flow + +- `GET` of page Markdown for editing with a `lang` parameter returns the + hybrid (not the raw original) when the article has that language. +- `PUT`/WS save with `lang` does **not** touch `node.chunks`; it diffs + against the hybrid that was served and appends a `Patch`. (Serve a hybrid + generation token with the editor payload so a save based on a stale hybrid + can be rebased or rejected — simplest: recompute the diff against the + *current* hybrid and accept best-effort, matching the patch philosophy.) +- Saving the primary-language version re-chunks the submitted Markdown and + updates `Data.chunks` / `node.chunks` — only genuinely new text lands in + the kanta change diff (see docs/migrate.md). + +### Explicitly out of scope for phase 2 + +- The machine translation itself: chunking output goes in, translated chunks + come back. A background job writes `trans` entries; this doc only defines + the storage key (`chunk_hash:lang`) and the merge semantics. +- Garbage collection of orphaned chunks/translations (see docs/migrate.md). +- sitemap.xml per-language entries; translated UI chrome; per-language + typographer options; multi-locale date/number formatting. diff --git a/docs/migrate.md b/docs/migrate.md new file mode 100644 index 0000000..46d9493 --- /dev/null +++ b/docs/migrate.md @@ -0,0 +1,142 @@ +# migrate_v3: content-addressed chunk storage + +Status: **final plan**. `migrate_v3` restructures how article text and +translations are stored, motivated by the localization model in +`docs/localization.md` (phase 2). Since it is a full migration, it is free to +break the current `Node.content: str | None` layout. + +## Goals + +- **Minimal change diffs.** kanta persists change diffs; editing one + paragraph of a long article must not rewrite the whole article string, and + a translation refresh must touch only the re-translated chunks. +- **Fast, simple lookup.** Everything heavy lives in flat + `dict[hash, content]` stores; ordering lives in `list[hash]`. No large + nested structures, no deep paths. +- **Path-independent text.** Chunks and their translations are keyed by + content hash, not by article path — the same paragraph (or menu title) + appearing in several articles is stored and translated once. Moving or + renaming an article touches nothing. + +## Design (chosen: global content-addressed stores) + +Original articles are *also* stored as chunks; everything — originals and +translations — lives in flat hash-keyed dicts. Costs accepted: rendering does +one dict lookup per chunk (trivial), orphaned hashes need occasional garbage +collection, and the editor save path re-chunks server-side (it already +diffs). The rejected alternatives: per-article nested `LangVersion` +structures (churn, duplication, whole-string originals) and a hybrid with +whole originals plus global translations (keeps the worst change-diff +property). + +## Target layout + +```python +class Node(msgspec.Struct, omit_defaults=True): + ... + #: Replaces `content: str | None`. None = pure category label; + #: a list (possibly empty) = a page, as ordered chunk hashes. + chunks: list[str] | None = None + #: Primary language of the article (BCP-47 base tag). "" = inherit + #: (nearest ancestor, front page last, site default "en" final). + language: str = "" + #: Chunk hashes the editor marked "do not translate" (always served + #: from the original). Presence-keys, value always True. + no_trans: dict[str, True] = {} + #: Languages this article is available in (besides its primary + #: language). Presence-keys, value always True — rendering, language + #: selection and hreflang alternates read this set instead of probing + #: the trans store chunk by chunk. Maintained by the writers (see + #: "Language index maintenance" below). + langs: dict[str, True] = {} + +class Data(msgspec.Struct): + ... + #: All original-language text, content-addressed: blake3(normalized, + #: digest 16) hex -> Markdown chunk. Shared by every article. + chunks: dict[str, str] = {} + #: Machine translations: f"{chunk_hash}:{lang}" -> translated Markdown. + #: Also used for node titles (hash of the title text). + trans: dict[str, str] = {} + #: User override patches per article and language: + #: f"{path}:{lang}" -> ordered patches (see localization.md). + patches: dict[str, list[Patch]] = {} +``` + +Notes: + +- **Article paths never carry a leading slash** in the DB or in lookup keys + (`"docs/setup"`, front page `""`); the leading slash is added only when + building hrefs. `migrate_v3` audits existing stored paths (translation + keys, analytics references, any path-valued fields) and normalizes them. +- **Titles are chunks too**, by hash only: the nav renderer looks up + `trans.get(f"{hash(node.title)}:{lang}")`. No separate title storage; + editing a title invalidates its translations automatically. +- **Per-hunk options** live in two places: *inherent* options are derived at + chunking time (code fences and HTML blocks are marked no-translate without + storing anything); *editor-set* flags are `node.no_trans` (keyed by chunk + hash, so a heavy edit silently drops the flag — acceptable and + self-healing). +- **Patch payloads stay inline** in `Patch.hunks` — patches are small by + construction (minimal server-computed diffs). If a pathological case shows + up, hunks can be hash-stored later without schema pain. + +## Language index maintenance (`node.langs`) + +`node.langs` is a denormalized index over the `trans`/`patches` stores so +that article rendering, `select_language`'s availability check, and hreflang +alternate links never enumerate chunks. It is written by whoever writes +translation data, in the same transaction: + +- **Translator job:** after writing `trans[f"{h}:{lang}"]` entries for an + article's chunks (or its title), set `node.langs[lang] = True`. +- **Translated-view save:** appending the first patch for `f"{path}:{lang}"` + sets `node.langs[lang] = True` (patches alone make the version exist). +- **Removals:** deleting a patch or GC'ing translations re-derives the key: + keep `lang` if any `trans` entry for the article's current chunks/title or + any patch remains, otherwise drop it. Stale `langs` keys are benign (an + advertised language that renders as the original), so removal can lag. + +## Render / save pipeline (summary) + +- **Render:** `text = "\n\n".join(chunks[h] for h in node.chunks)` for the + original; for language `L` (only ever attempted when `L in node.langs`), + per chunk `trans[f"{h}:{L}"]` unless missing or `h in node.no_trans`, + falling back to `chunks[h]`; then apply `patches.get(f"{path}:{L}", [])` + in order (per-hunk, best effort); then `markdown.render` as today. All of + this assembles the `Translation` the phase-1 plumbing already consumes. +- **Availability:** `available_languages(path)` = `sorted(node.langs)`; + hreflang alternates and `?lang=` handling use exactly this set. +- **Save (primary language):** server re-chunks the submitted Markdown, + inserts new hashes into `Data.chunks`, replaces `node.chunks`. Unchanged + chunks keep their hashes — only genuinely new text lands in the diff. +- **Save (translated view):** diff against the served hybrid, append a + `Patch` under `patches[f"{path}:{lang}"]`; `node.chunks` untouched. +- **Invalidate:** any write to `chunks` / `trans` / `patches` calls + `_invalidate_pages()`. + +## migrate_v3 steps + +1. Walk `menu`; for every node with a string `content`: + `chunks = chunk_markdown(content)`; write each into the new `chunks` + store; replace the field with the hash list (`None` stays `None`). +2. Initialize empty `chunks` / `trans` / `patches` stores. +3. Normalize stored paths: strip leading slashes anywhere paths are keys or + values. +4. `language`, `no_trans` and `langs` need nothing — struct defaults cover + them (`langs` starts empty; the translator job fills it as translations + land). + +Chunking must be deterministic and shared with render/save, so +`chunk_markdown` + `chunk_key` live in `pagerite/i18n.py` (or a small +`pagerite/chunks.py`) and are imported by both `migrations.py` and +`views.py`/`app.py`. + +## Garbage collection (later, manual or idle-time) + +Orphaned entries accumulate: chunks no longer referenced by any +`node.chunks`/`node.title`, translations whose chunk hash is orphaned, patch +hunks that never match. All are harmless (never read). A GC pass is a single +tree walk collecting live hashes, then deleting the rest from `chunks` and +`trans`; patches whose every hunk is stale get pruned. Not part of +migrate_v3. diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js index 1b9351b..5b95d6f 100644 --- a/frontend/src/pagerite.js +++ b/frontend/src/pagerite.js @@ -34,6 +34,24 @@ import "overlayscrollbars/overlayscrollbars.css"; }); } + // --- Language override (?lang=) --------------------------------------- + // /page?lang=fi serves a translated, indexable version (each language is + // its own canonical). Restore the pretty URL on load but remember the + // language: all fetch-navigation and preload requests below send it as + // Accept-Language, so the chosen language sticks for the session of + // clicks. A full refresh or shared link resets to automatic selection + // (the browser's own Accept-Language). See docs/localization.md. + const langParam = new URL(location.href).searchParams.get("lang"); + if (langParam) { + const url = new URL(location.href); + url.searchParams.delete("lang"); + history.replaceState(history.state, "", url); + } + const pageHeaders = langParam ? { "Accept-Language": langParam } : {}; + // The in-memory page cache is keyed per language: the same pathname holds + // different HTML for each selected language. + const cacheKey = (pathname) => (langParam ? `${langParam}|${pathname}` : pathname); + // Regions every page has. #sidebar is NOT among them: it is omitted // entirely when the section has no sub-navigation, and handled below. const REGIONS = ["page-banner", "nav", "main"]; @@ -352,9 +370,9 @@ import "overlayscrollbars/overlayscrollbars.css"; // received it as the document (re-fetching would be redundant, and // browser heuristics may send it without if-none-match, defeating the // conditional request); it enters the cache when navigated to. - const pageCache = new Map(); // pathname -> HTML text + const pageCache = new Map(); // cacheKey(pathname) -> HTML text addEventListener("pagerite:page-fetched", (ev) => { - pageCache.set(new URL(ev.detail.url, location.href).pathname, ev.detail.html); + pageCache.set(cacheKey(new URL(ev.detail.url, location.href).pathname), ev.detail.html); }); // Editors mutate site-wide state (theme, structure, headings, banners), @@ -380,14 +398,14 @@ import "overlayscrollbars/overlayscrollbars.css"; urls.add(a.pathname); } for (const url of urls) { - if (pageCache.has(url)) continue; + if (pageCache.has(cacheKey(url))) continue; // x-pagerite-preload: idle cache warm-up, not a page view — the // server excludes these GETs from analytics (the navigation message // sent on actual navigation does the counting). - fetch(url, { headers: { "x-pagerite-preload": "1" } }) + fetch(url, { headers: { "x-pagerite-preload": "1", ...pageHeaders } }) .then((r) => (r.ok && (r.headers.get("content-type") || "").includes("text/html") ? r.text() : "")) - .then((html) => { if (html) pageCache.set(url, html); }) + .then((html) => { if (html) pageCache.set(cacheKey(url), html); }) .catch(() => {}); } } @@ -697,12 +715,12 @@ import "overlayscrollbars/overlayscrollbars.css"; teardownAnalytics(); let doc; let finalUrl = url; - const cached = !editing && pageCache.get(new URL(url, location.href).pathname); + const cached = !editing && pageCache.get(cacheKey(new URL(url, location.href).pathname)); if (cached) { doc = new DOMParser().parseFromString(cached, "text/html"); } else { try { - const res = await fetch(url); + const res = await fetch(url, { headers: pageHeaders }); const type = res.headers.get("content-type") || ""; if (!res.ok || !type.includes("text/html")) throw new Error("not a page"); // Reflect any redirect the server issued. @@ -710,7 +728,7 @@ import "overlayscrollbars/overlayscrollbars.css"; const html = await res.text(); // Populate the cache too, so returning here (back/forward, or a // self-link in the nav) is served from memory. - pageCache.set(new URL(finalUrl, location.href).pathname, html); + pageCache.set(cacheKey(new URL(finalUrl, location.href).pathname), html); doc = new DOMParser().parseFromString(html, "text/html"); } catch { location.href = url; // fall back to a normal navigation diff --git a/pagerite/app.py b/pagerite/app.py index 0ef5d0b..0e6af55 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -47,7 +47,7 @@ from mediapreview import dispatch from pydantic import BaseModel from zstandard import ZstdCompressor -from pagerite import analytics, seed, views +from pagerite import analytics, i18n, seed, views from pagerite.__main__ import DEVMODE from pagerite.data import ( Data, @@ -381,10 +381,13 @@ class FileStore: file_store = FileStore(FILES_DIR) -def _render_html(kind: str, path: str, base_url: str) -> str: +def _render_html(kind: str, path: str, base_url: str, lang: str = i18n.ORIGINAL_LANGUAGE) -> str: """Render one of the generated pages (see _html_response).""" if kind == "page": - return views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition) + # A selected language without an actual translation renders the + # original (translation is None = English; see docs/localization.md). + translation = i18n.get_translation(path, lang) if lang != i18n.ORIGINAL_LANGUAGE else None + return views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition, lang=lang, translation=translation) if kind == "category": return views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) if kind == "not-found": @@ -406,13 +409,14 @@ def _invalidate_pages() -> None: @lru_cache(maxsize=128) -def _cached_body(kind: str, path: str, base_url: str, zstd: bool) -> bytes: +def _cached_body(kind: str, path: str, base_url: str, zstd: bool, lang: str = i18n.ORIGINAL_LANGUAGE) -> bytes: """Rendered page body; cleared by _invalidate_pages on any - content/settings change. base_url feeds the social meta URLs and zstd + content/settings change. base_url feeds the social meta URLs, zstd selects the stored encoding (both variants are cached rather than - re-compressed). + re-compressed) and lang the selected language (not the raw + Accept-Language header, which would blow up the cache key space). """ - body = _render_html(kind, path, base_url).encode() + body = _render_html(kind, path, base_url, lang).encode() return _zstd.compress(body) if zstd else body @@ -423,6 +427,7 @@ def _html_response( status_code: int = 200, headers: dict | None = None, etag: bool = False, + lang: str = i18n.ORIGINAL_LANGUAGE, ) -> Response: """Response for a generated page, zstd-compressed when the client accepts it (no gzip fallback). @@ -444,14 +449,15 @@ def _html_response( # localhost (varying ports) fall back to the request's own base URL. base_url = SITE_URL or str(request.base_url).rstrip("/") if DEVMODE: - identity = _render_html(kind, path, base_url).encode() + identity = _render_html(kind, path, base_url, lang).encode() body = _zstd.compress(identity) if zstd else identity else: - identity = _cached_body(kind, path, base_url, False) - body = _cached_body(kind, path, base_url, True) if zstd else identity + identity = _cached_body(kind, path, base_url, False, lang) + body = _cached_body(kind, path, base_url, True, lang) if zstd else identity h = dict(headers or {}) - if zstd: - h["vary"] = "accept-encoding" + # Content varies by language (Accept-Language selects a translation) + # and by encoding; keep caches from mixing either representation. + h["vary"] = "accept-language" + (", accept-encoding" if zstd else "") if etag: tag = f'"{blake3.blake3(identity).hexdigest()[:32]}"' h["etag"] = tag @@ -1509,13 +1515,22 @@ async def show_page(request: Request, path: str) -> Response: chain = resolve(data.menu, path) node = chain[-1] if chain else None if node is not None and node.published and node.content is not None: + # Language selection (docs/localization.md): ?lang= wins when a + # translation exists, else header logic. Analytics keep the raw + # Accept-Language header regardless of the selection. + languages = i18n.available_languages(path) + lang = i18n.select_language( + request.query_params.get("lang"), + accept_language, + lambda l: l in languages, + ) # no-cache forbids serving a stored page without revalidation # (browsers would otherwise cache heuristically and serve stale # pages, e.g. after a theme change). In-session speed instead comes # from pagerite.js's in-memory page cache (preload everything, never # fetch on navigation); the ETag just makes those one-time preload # fetches and any revalidation cheap. - etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}"' + etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}l{lang}"' if request.headers.get("if-none-match") == etag: return Response(status_code=304) if _is_trackable_path(path): @@ -1530,6 +1545,7 @@ async def show_page(request: Request, path: str) -> Response: "last-modified": _http_date(node.modified), "cache-control": "no-cache", }, + lang=lang, ) if node is not None and node.published and node.content is None: # Category label without a landing page: placeholder with the pen diff --git a/pagerite/i18n.py b/pagerite/i18n.py new file mode 100644 index 0000000..126d8dd --- /dev/null +++ b/pagerite/i18n.py @@ -0,0 +1,89 @@ +"""Localization: language selection and translation access. + +See docs/localization.md. The database keeps only the original language +(English); translations of page Markdown and navigation titles are consumed +through get_translation(), with per-node fallback to the original titles. +""" + +from collections.abc import Callable + +import msgspec + +#: Language of the database originals (and the default ). +ORIGINAL_LANGUAGE = "en" + + +class Translation(msgspec.Struct, omit_defaults=True): + """Translated content for one page and language. + + ``markdown`` is the translated page source in the same format as the + original (None = keep the original Markdown); ``titles`` maps node paths + (top-level slug, then slash-joined) to translated navigation titles, so a + partially translated tree still renders with per-node English fallback. + """ + + markdown: str | None = None + titles: dict[str, str] = {} + + +def parse_accept_language(header: str) -> list[str]: + """Accept-Language header as an ordered, deduped list of base subtags. + + q-values are deliberately ignored: all known implementations send the + header in order of preference. Region tags normalize to their base + subtag (fi-FI -> fi); "*" and empties are dropped. + """ + langs = [] + for part in header.split(","): + tag = part.split(";", 1)[0].strip().lower().partition("-")[0] + if tag and tag != "*" and tag not in langs: + langs.append(tag) + return langs + + +def select_language( + query_lang: str | None, + accept_language: str | None, + is_available: Callable[[str], bool], + original: str = ORIGINAL_LANGUAGE, +) -> str: + """The language to serve (see docs/localization.md). + + 1. ``?lang=`` wins when a translation exists for it (otherwise falls + through to the header logic). + 2. The original language anywhere in the header list wins — an AI + translation is strictly worse than the original for anyone who has + English configured at all. + 3. Otherwise the first header language with an available translation. + 4. Fall back to the original. + """ + if query_lang: + tag = query_lang.strip().lower().partition("-")[0] + if tag == original or (tag and is_available(tag)): + return tag + langs = parse_accept_language(accept_language or "") + if original in langs: + return original + for lang in langs: + if lang != original and is_available(lang): + return lang + return original + + +def get_translation(path: str, lang: str) -> Translation | None: + """The translation of the page at ``path`` for ``lang``, or None. + + TODO: stub for the AI-translation track, which will provide the real + implementation (storage, generation, invalidation on source changes). + The CMS only consumes this function and available_languages(). + """ + return None + + +def available_languages(path: str) -> list[str]: + """Languages with a translation for the page at ``path`` (besides the + original). Drives language selection and hreflang alternate links. + + TODO: stub, together with get_translation(). + """ + return [] diff --git a/pagerite/views.py b/pagerite/views.py index d61ee52..6cb986c 100644 --- a/pagerite/views.py +++ b/pagerite/views.py @@ -24,7 +24,9 @@ import re from html5tagger import HTML, Document, E, Template from platformdirs import site_data_dir, user_data_path +from pagerite import i18n from pagerite.data import Node, prettify, resolve, sorted_nodes +from pagerite.i18n import Translation from pagerite.markdown import render SITE_NAME = "Pagerite" @@ -309,6 +311,8 @@ def _layout( transition: str = "cube", favicon: str = "", social: dict[str, str] | None = None, + lang: str = i18n.ORIGINAL_LANGUAGE, + alternates: list[tuple[str, str]] = (), ) -> Template: """Page layout template with standard assets and ES-module scripts. @@ -333,8 +337,12 @@ def _layout( ``social`` maps meta keys to contents: ``og:*``/``article:*`` go out as property attributes, everything else (description, twitter:*) as name. + + ``lang`` is the served language for ; ``alternates`` holds + (hreflang, href) pairs for the other language versions of the page, + emitted as (see docs/localization.md). """ - doc = Document(E.Title, lang="en") + doc = Document(E.Title, lang=lang) # Responsive layout (see the 48rem breakpoint in pagerite.css) needs # the real device width, not the default 980px layout viewport. doc.meta(name="viewport", content="width=device-width, initial-scale=1") @@ -346,6 +354,8 @@ def _layout( doc.link(rel="canonical", href=value) else: doc.meta(name=key, content=value) + for hreflang, href in alternates: + doc.link(rel="alternate", hreflang=hreflang, href=href) # A custom favicon (from the site editor) is linked explicitly; without # one, browsers fall back to the build's /favicon.ico by convention. if favicon: @@ -452,14 +462,20 @@ def _brand_link(brand: str, brand_html: str = "") -> HTML: return HTML(str(E.a(brand, href="/", id="brand"))) if brand else HTML("") -def _title(slug: str, node: Node) -> str: - """Menu label: the configured title, prettified slug, "Home" fallback.""" +def _title(slug: str, node: Node, translation: Translation | None = None, path: str = "") -> str: + """Menu label: the configured title, prettified slug, "Home" fallback. + + With a translation, its title map (keyed by node path) wins, falling + back per node to the original English title. + """ + if translation and (t := translation.titles.get(path)): + return t return node.title or prettify(slug) or "Home" def _nav_link( doc, menu: dict[str, Node], node: Node, path: str, current: str, - ancestors_current: bool = True, + ancestors_current: bool = True, translation: Translation | None = None, ) -> None: """Render one
  • linking the node. Category labels (no content of their own — None, or empty markdown as left by the site editor's @@ -474,13 +490,13 @@ def _nav_link( if not node.content and (leaf := first_leaf(menu, path)) is not None: href = f"/{leaf}" doc.li.a( - _title(path.rpartition("/")[2], node), + _title(path.rpartition("/")[2], node, translation, path), href=href, **{"class": "current"} if is_current else {}, ) -def nav_html(menu: dict[str, Node], current: str) -> HTML: +def nav_html(menu: dict[str, Node], current: str, translation: Translation | None = None) -> HTML: """Render the contents of the #nav element for the current path. Top-level items in menu order; the front page (slug "", href "/") @@ -491,11 +507,11 @@ def nav_html(menu: dict[str, Node], current: str) -> HTML: with nav: for slug, node in sorted_nodes(menu): if node.published: - _nav_link(nav, menu, node, slug, current) + _nav_link(nav, menu, node, slug, current, translation=translation) return HTML(str(nav)) -def sidebar_html(menu: dict[str, Node], current: str) -> HTML: +def sidebar_html(menu: dict[str, Node], current: str, translation: Translation | None = None) -> HTML: """Render the #sidebar element for the current path (empty when none). The sidebar is the current main level section's sub-navigation: the @@ -529,20 +545,20 @@ def sidebar_html(menu: dict[str, Node], current: str) -> HTML: nav = E.ul with nav: for slug, child in items: - _sidebar_item(nav, menu, child, f"{section}/{slug}", current) + _sidebar_item(nav, menu, child, f"{section}/{slug}", current, translation) return HTML(str(E.aside(nav, id="sidebar"))) -def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: str) -> None: +def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: str, translation: Translation | None = None) -> None: """One sidebar
  • : the node link, with its published children as a nested list (third level and deeper, recursively).""" - _nav_link(doc, menu, node, path, current, ancestors_current=False) + _nav_link(doc, menu, node, path, current, ancestors_current=False, translation=translation) sub = [(s, c) for s, c in sorted_nodes(node.children) if c.published] if sub: # doc.li.a(...) above left the
  • open for nesting. with doc.ul: for slug, child in sub: - _sidebar_item(doc, menu, child, f"{path}/{slug}", current) + _sidebar_item(doc, menu, child, f"{path}/{slug}", current, translation) def first_leaf(menu: dict[str, Node], path: str) -> str | None: @@ -674,16 +690,24 @@ def banner_source(menu: dict[str, Node], path: str) -> str | None: return None -def page_content(menu: dict[str, Node], path: str) -> HTML: +def page_content(menu: dict[str, Node], path: str, translation: Translation | None = None) -> HTML: """Render the contents of the #main element for a page. A page with published children (a category page) lists them as cards - after the markdown content. + after the markdown content. With a translation, its Markdown goes + through the same render pipeline; missing pieces (markdown=None, absent + title entries) fall back to the original. """ node = resolve(menu, path)[-1] + content = node.content or "" + title = node.title + if translation: + if translation.markdown is not None: + content = translation.markdown + title = _title(path.rpartition("/")[2], node, translation, path) if node.title else title # The title is injected into the markdown (as # title when it has no # h1 of its own), so title and content render as one article. - rendered = render(node.content or "", path, node.created, node.modified, title=node.title) + rendered = render(content, path, node.created, node.modified, title=title) # Long articles get .multicol: the article column cap lifts (see the # #content grid in pagerite.css) and the .cols segments lay out in at # most two columns. The html is already segmented by render() — the @@ -691,11 +715,11 @@ def page_content(menu: dict[str, Node], path: str) -> HTML: doc = E.article(class_="multicol") if rendered.multicol else E.article with doc: doc(HTML(rendered.html)) - _cards(doc, menu, node, path) + _cards(doc, menu, node, path, translation) return HTML(str(doc)) -def _cards(doc, menu: dict[str, Node], node: Node, path: str) -> None: +def _cards(doc, menu: dict[str, Node], node: Node, path: str, translation: Translation | None = None) -> None: """Card stacks of the node's published children (nothing when childless). One column per direct child, all in a single full-width row (the .wide @@ -721,7 +745,7 @@ def _cards(doc, menu: dict[str, Node], node: Node, path: str) -> None: continue with doc.div(class_="stack"): for epath, enode in entries: - _card(doc, enode, epath) + _card(doc, enode, epath, translation) def _walk(node: Node, path: str): @@ -735,9 +759,13 @@ def _walk(node: Node, path: str): yield from _walk(child, f"{path}/{slug}") -def _card(doc, node: Node, path: str) -> None: +def _card(doc, node: Node, path: str, translation: Translation | None = None) -> None: """One card in a stack: cover + title, plus the description when the - page has no image (its card shows a gradient cover instead).""" + page has no image (its card shows a gradient cover instead). + + Cover/description heuristics run on the original Markdown even when + translated (only this page's own Markdown is translated); the title + uses the translation's title map.""" image = description = "" if node.content: html = render(node.content, path, node.created, node.modified).html @@ -749,7 +777,7 @@ def _card(doc, node: Node, path: str) -> None: doc.span(class_="cover", style=f'background-image: url("{image}")') else: doc.span(class_="cover") - doc.span(_title(path.rpartition("/")[2], node), class_="title") + doc.span(_title(path.rpartition("/")[2], node, translation, path), class_="title") if description: doc.span(description, class_="desc") @@ -833,6 +861,7 @@ def _share_media(html: str, base_url: str) -> tuple[str, str]: def _social_meta( node: Node, path: str, title: str, html: str, brand: str, base_url: str, + lang: str = i18n.ORIGINAL_LANGUAGE, ) -> dict[str, str]: """Open Graph/Twitter/SEO meta tags for a content page. @@ -842,6 +871,9 @@ def _social_meta( representative figure. Absolute URLs are built from the request's base (social scrapers cannot use relative ones). + When a translation is served, the canonical URL includes the ?lang= + query: each language version is its own canonical (docs/localization.md). + ``twitter:image`` pins extension-less store links to the ``.webp`` variant: X only honors WebP via twitter:image (not og:image) and its scraper cannot be trusted to negotiate via Accept. @@ -854,7 +886,7 @@ def _social_meta( ) return { "description": text, - "canonical": url, + "canonical": f"{url}?lang={lang}" if url and lang != i18n.ORIGINAL_LANGUAGE else url, "og:type": "article", "og:title": title, "og:description": text, @@ -879,21 +911,39 @@ def render_page( brand_html: str = "", base_url: str = "", transition: str = "cube", + lang: str = i18n.ORIGINAL_LANGUAGE, + translation: Translation | None = None, ) -> str: - """Render a full HTML page for the slug path.""" + """Render a full HTML page for the slug path. + + ``lang``/``translation`` serve a translated version (see + docs/localization.md): None translation = the English original. + """ node = resolve(menu, path)[-1] - title = _title(path.rpartition("/")[2], node) - main = page_content(menu, path) - social = _social_meta(node, path, title, str(main), brand, base_url) + if translation is None: + lang = i18n.ORIGINAL_LANGUAGE + title = _title(path.rpartition("/")[2], node, translation, path) + main = page_content(menu, path, translation) + social = _social_meta(node, path, title, str(main), brand, base_url, lang) + # hreflang alternates: every other language version (with ?lang=) plus + # x-default for the plain URL. Emitted only when translations exist. + alternates = [] + if base_url: + available = i18n.available_languages(path) + alternates = [ + (l, f"{base_url}/{path}?lang={l}") for l in available if l != lang + ] + if available: + alternates.append(("x-default", f"{base_url}/{path}")) return str( _layout( *_page_assets(), custom_css, theme, banner_design(menu, path, theme), - transition, favicon, social, + transition, favicon, social, lang, alternates, )( Title=f"{title} – {brand}" if brand else title, Brand=_brand_link(brand, brand_html), - Nav=nav_html(menu, path), - Sidebar=sidebar_html(menu, path), + Nav=nav_html(menu, path, translation), + Sidebar=sidebar_html(menu, path, translation), Banner=banner_html(menu, path, theme), Main=main, ),