From 218313ba7a9ced8aca38382aa9f8d2a6eb0afe34 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:14:10 +0000 Subject: [PATCH 01/31] Phase 1: content negotiation (?lang=, Accept-Language) + localization/migration design docs - pagerite/i18n.py: parse_accept_language, select_language, Translation struct, get_translation/available_languages stubs - app.py: language threaded through show_page/_render_html/_cached_body, ETag and Vary: accept-language - views.py: , translated nav titles with fallback, canonical with ?lang=, hreflang alternates - pagerite.js: strip ?lang= via replaceState, send it as Accept-Language on fetch-navigation/preloads, lang-aware page cache - docs/localization.md: negotiation + URL scheme + phase-2 fragment model - docs/migrate.md: migrate_v3 content-addressed chunk storage plan --- docs/localization.md | 219 +++++++++++++++++++++++++++++++++++++++ docs/migrate.md | 142 +++++++++++++++++++++++++ frontend/src/pagerite.js | 34 ++++-- pagerite/app.py | 42 +++++--- pagerite/i18n.py | 89 ++++++++++++++++ pagerite/views.py | 108 +++++++++++++------ 6 files changed, 584 insertions(+), 50 deletions(-) create mode 100644 docs/localization.md create mode 100644 docs/migrate.md create mode 100644 pagerite/i18n.py diff --git a/docs/localization.md b/docs/localization.md new file mode 100644 index 0000000..704336c --- /dev/null +++ b/docs/localization.md @@ -0,0 +1,219 @@ +# Localization + +Pages are served in the visitor's language based on a `?lang=` query +parameter or the `Accept-Language` header. + +- **Phase 1 (implemented):** negotiation, URL scheme, caching, rendering + plumbing. Translations are consumed through a stub interface; the database + still holds only the original language. +- **Phase 2 (final plan):** gettext-style fragment storage in the + database — machine-translated chunks plus user override patches, assembled + at render time. Storage details in `docs/migrate.md`. + +## Phase 1: negotiation and URLs + +### Language selection + +Deliberately simple — **q-values are ignored**: + +- All known `Accept-Language` implementations send the header **in order of + preference**, so we parse it as an ordered list and never reorder. +- Selection rule (`select_language` in `pagerite/i18n.py`): + 1. If `?lang=` is present, use it (if a translation exists; otherwise + fall through to header logic). + 2. If the article's original language appears anywhere in the header list, + use the **original**. Rationale: an AI translation is strictly worse + than the original for anyone who has that language configured at all + (e.g. `fi-FI, fi, en-US, en` gets English, not machine-translated + Finnish). + 3. Otherwise walk the header list in order and use the first language for + which a translation exists. + 4. Fall back to the original. + +Region tags normalize to their base subtag (`fi-FI` → `fi`). + +### URLs: pretty for users, indexable for search engines + +- Canonical URLs stay pretty (`/some-page`). Each language version is + addressable as `/some-page?lang=fi` so search engines can index them. +- `` points to the page **itself including the query** + (each language version is its own canonical). +- `` entries point to every other language + version (with `?lang=`), plus `x-default` for the plain URL. +- On page load, pagerite.js removes the `?lang=` query via + `history.replaceState` (restoring the pretty URL) but remembers the language + in a JS variable. All fetch-navigation and preloads it performs afterwards + send that language in the `Accept-Language` header, so the chosen language + sticks for the session of clicks. +- A full page refresh or a shared link resets to automatic selection (header + only). This gives a clean one-time override without cookies. + +### Response correctness + +- Content responses carry `Vary: accept-language` (added to the existing + `accept-encoding` vary). +- `_cached_body` and the page ETag include the **selected language** (not the + raw header, which would blow up the cache key space). +- `` reflects the served language. + +### Rendering + +- The translated Markdown goes through the same `markdown.render` pipeline. +- Navigation/sidebar titles come from the translation's title map, with + per-node fallback to the original title (a partially translated tree must + still render). +- Fixed UI strings ("Not Found" etc.) and the editor UI stay English for now. +- The markdown typographer (SmartyPants) is English-centric; per-language + typographer options are a possible follow-up, not blocking. + +## Phase 2: fragment-based translation storage (draft) + +Phase 1 assumed whole-page translated Markdown delivered from outside. The +refined model is gettext-style: an article has **one primary version** (its +`content`, in its own language) plus, per target language, **machine +fragments** (translated chunks of Markdown) and **user patches** (minimal +editor overrides). Both are stored in the database and assembled into the +served Markdown at render time. + +### The scenario this must handle + +1. Article written in English. +2. Machine-translated into Spanish → fragments stored. +3. Editor fixes one Spanish paragraph and changes a link elsewhere to point + at a Spanish resource → user patch hunks stored. +4. English article edited → the edited chunk's key changes; its Spanish + fragment no longer matches. +5. Page requested before the machine translation refreshes → served as a + **hybrid**: old fragments for unchanged chunks, plain English for the + edited chunk. User patches are attempted against this hybrid, best effort, + each hunk independently: the text fix is stale (its search text no longer + exists) and silently skipped; the link change still applies even though + the link sits in the now-English paragraph. +6. Machine translation refreshes → full Spanish again, with both patch hunks + applying. + +### Chunks + +`chunk_markdown(markdown)` splits the source into block-level chunks — +blank-line-separated blocks: headings, paragraphs, code fences (kept whole), +list blocks, tables, HTML blocks. A chunk's identity is its **source text**, +gettext-msgid style: + +```python +chunk_key = blake3(normalize(chunk_text), digest_size=16).hex() +``` + +(`normalize`: strip trailing whitespace per line, collapse surrounding blank +lines — so whitespace-only source edits don't invalidate translations.) + +Consequences: + +- Editing the English source invalidates exactly the edited chunks; all + other fragments keep applying. Stale fragments are simply never referenced + again and can be garbage-collected lazily (or left; they are tiny). +- No explicit "source version" bookkeeping is needed — staleness falls out + of the keys. + +### User patches + +Editors always edit **full Markdown** in the existing editor UX — never +fragments. When editing a translated view (`?lang=es`), the editor is loaded +with the *current hybrid Markdown*; on save, the server computes a minimal +diff against that hybrid and stores it as a patch: + +```python +class Patch(msgspec.Struct, omit_defaults=True): + """One editing session's overrides, applied independently per hunk.""" + + hunks: list[tuple[str, str]] = [] # (search, replace) on hybrid Markdown +``` + +Hunks are produced from `difflib.SequenceMatcher` on the hybrid vs. the +edited text at block granularity: each `replace`/`delete`/`insert` opcode +becomes one `(search, replace)` pair, with the preceding block's tail as +left context for `insert` (pure inserts have empty search context otherwise). +Application is dead simple: + +```python +def apply_patch(hybrid: str, patch: Patch) -> str: + for search, replace in patch.hunks: + if search and search in hybrid: + hybrid = hybrid.replace(search, replace, 1) + # missing search text = stale hunk -> silently skipped + return hybrid +``` + +Per-hunk independence is the robustness property from the scenario: a stale +text fix does not block a still-valid link change. Patches are stored as an +ordered list and applied in order. + +### Storage + +Full storage design and the `migrate_v3` restructuring live in +`docs/migrate.md`. The short version, as it concerns this document: + +- Originals **and** translations are content-addressed text chunks in flat + stores: `Data.chunks: dict[hash, str]` and + `Data.trans: dict[f"{chunk_hash}:{lang}", str]` — path-independent, so + repeated paragraphs and menu titles are translated once and article moves + touch nothing. `Node.chunks: list[hash]` gives each article its order. +- `Node` gains **`language: str = ""`**, inherited down the tree like + `banner` (empty = nearest ancestor, front page last, site default `en` + final). `select_language` and `` use the resolved value instead + of the global `ORIGINAL_LANGUAGE` constant. + - **Known weakness:** changing a page's (or subtree's) `language` after + translations exist mis-keys everything — translations are keyed by + *source* chunks, so old entries silently stop matching and user patches + (searching for old-hybrid text) mostly go stale. That is acceptable: + the orphaned data is harmless and translations regenerate. We do not + migrate translations across a language change. +- Article paths are stored and keyed **without leading slashes** + (`"docs/setup"`, front page `""`); slashes are added only in hrefs. + +### Render pipeline (replaces the phase-1 `get_translation` stub) + +```python +def get_translation(path, lang, data) -> Translation | None: + if lang not in node.langs: + return None + hybrid = "\n\n".join( + chunks[h] if h in node.no_trans else trans.get(f"{h}:{lang}", chunks[h]) + for h in node.chunks + ) + for patch in data.patches.get(f"{path}:{lang}", []): + hybrid = apply_patch(hybrid, patch) + return Translation(markdown=hybrid, titles=title_map(data, lang)) +``` + +- Availability is an article-level index: `node.langs: dict[lang, True]`, + maintained by the translation writers (translator job, patch saves) in the + same transaction as their data writes — rendering, language selection and + hreflang never probe the `trans` store chunk by chunk. A stale key is + benign (the "translation" just renders as the original). +- `titles` for nav/sidebar/cards: each node's translated title is + `trans.get(f"{hash(node.title)}:{lang}")` with per-node fallback — one dict + lookup per nav item at render time. +- Cache invalidation: writes to `chunks` / `trans` / `patches` (translator, + editor saves) call `_invalidate_pages()`, same as content writes. + +### Editor flow + +- `GET` of page Markdown for editing with a `lang` parameter returns the + hybrid (not the raw original) when the article has that language. +- `PUT`/WS save with `lang` does **not** touch `node.chunks`; it diffs + against the hybrid that was served and appends a `Patch`. (Serve a hybrid + generation token with the editor payload so a save based on a stale hybrid + can be rebased or rejected — simplest: recompute the diff against the + *current* hybrid and accept best-effort, matching the patch philosophy.) +- Saving the primary-language version re-chunks the submitted Markdown and + updates `Data.chunks` / `node.chunks` — only genuinely new text lands in + the kanta change diff (see docs/migrate.md). + +### Explicitly out of scope for phase 2 + +- The machine translation itself: chunking output goes in, translated chunks + come back. A background job writes `trans` entries; this doc only defines + the storage key (`chunk_hash:lang`) and the merge semantics. +- Garbage collection of orphaned chunks/translations (see docs/migrate.md). +- sitemap.xml per-language entries; translated UI chrome; per-language + typographer options; multi-locale date/number formatting. diff --git a/docs/migrate.md b/docs/migrate.md new file mode 100644 index 0000000..46d9493 --- /dev/null +++ b/docs/migrate.md @@ -0,0 +1,142 @@ +# migrate_v3: content-addressed chunk storage + +Status: **final plan**. `migrate_v3` restructures how article text and +translations are stored, motivated by the localization model in +`docs/localization.md` (phase 2). Since it is a full migration, it is free to +break the current `Node.content: str | None` layout. + +## Goals + +- **Minimal change diffs.** kanta persists change diffs; editing one + paragraph of a long article must not rewrite the whole article string, and + a translation refresh must touch only the re-translated chunks. +- **Fast, simple lookup.** Everything heavy lives in flat + `dict[hash, content]` stores; ordering lives in `list[hash]`. No large + nested structures, no deep paths. +- **Path-independent text.** Chunks and their translations are keyed by + content hash, not by article path — the same paragraph (or menu title) + appearing in several articles is stored and translated once. Moving or + renaming an article touches nothing. + +## Design (chosen: global content-addressed stores) + +Original articles are *also* stored as chunks; everything — originals and +translations — lives in flat hash-keyed dicts. Costs accepted: rendering does +one dict lookup per chunk (trivial), orphaned hashes need occasional garbage +collection, and the editor save path re-chunks server-side (it already +diffs). The rejected alternatives: per-article nested `LangVersion` +structures (churn, duplication, whole-string originals) and a hybrid with +whole originals plus global translations (keeps the worst change-diff +property). + +## Target layout + +```python +class Node(msgspec.Struct, omit_defaults=True): + ... + #: Replaces `content: str | None`. None = pure category label; + #: a list (possibly empty) = a page, as ordered chunk hashes. + chunks: list[str] | None = None + #: Primary language of the article (BCP-47 base tag). "" = inherit + #: (nearest ancestor, front page last, site default "en" final). + language: str = "" + #: Chunk hashes the editor marked "do not translate" (always served + #: from the original). Presence-keys, value always True. + no_trans: dict[str, True] = {} + #: Languages this article is available in (besides its primary + #: language). Presence-keys, value always True — rendering, language + #: selection and hreflang alternates read this set instead of probing + #: the trans store chunk by chunk. Maintained by the writers (see + #: "Language index maintenance" below). + langs: dict[str, True] = {} + +class Data(msgspec.Struct): + ... + #: All original-language text, content-addressed: blake3(normalized, + #: digest 16) hex -> Markdown chunk. Shared by every article. + chunks: dict[str, str] = {} + #: Machine translations: f"{chunk_hash}:{lang}" -> translated Markdown. + #: Also used for node titles (hash of the title text). + trans: dict[str, str] = {} + #: User override patches per article and language: + #: f"{path}:{lang}" -> ordered patches (see localization.md). + patches: dict[str, list[Patch]] = {} +``` + +Notes: + +- **Article paths never carry a leading slash** in the DB or in lookup keys + (`"docs/setup"`, front page `""`); the leading slash is added only when + building hrefs. `migrate_v3` audits existing stored paths (translation + keys, analytics references, any path-valued fields) and normalizes them. +- **Titles are chunks too**, by hash only: the nav renderer looks up + `trans.get(f"{hash(node.title)}:{lang}")`. No separate title storage; + editing a title invalidates its translations automatically. +- **Per-hunk options** live in two places: *inherent* options are derived at + chunking time (code fences and HTML blocks are marked no-translate without + storing anything); *editor-set* flags are `node.no_trans` (keyed by chunk + hash, so a heavy edit silently drops the flag — acceptable and + self-healing). +- **Patch payloads stay inline** in `Patch.hunks` — patches are small by + construction (minimal server-computed diffs). If a pathological case shows + up, hunks can be hash-stored later without schema pain. + +## Language index maintenance (`node.langs`) + +`node.langs` is a denormalized index over the `trans`/`patches` stores so +that article rendering, `select_language`'s availability check, and hreflang +alternate links never enumerate chunks. It is written by whoever writes +translation data, in the same transaction: + +- **Translator job:** after writing `trans[f"{h}:{lang}"]` entries for an + article's chunks (or its title), set `node.langs[lang] = True`. +- **Translated-view save:** appending the first patch for `f"{path}:{lang}"` + sets `node.langs[lang] = True` (patches alone make the version exist). +- **Removals:** deleting a patch or GC'ing translations re-derives the key: + keep `lang` if any `trans` entry for the article's current chunks/title or + any patch remains, otherwise drop it. Stale `langs` keys are benign (an + advertised language that renders as the original), so removal can lag. + +## Render / save pipeline (summary) + +- **Render:** `text = "\n\n".join(chunks[h] for h in node.chunks)` for the + original; for language `L` (only ever attempted when `L in node.langs`), + per chunk `trans[f"{h}:{L}"]` unless missing or `h in node.no_trans`, + falling back to `chunks[h]`; then apply `patches.get(f"{path}:{L}", [])` + in order (per-hunk, best effort); then `markdown.render` as today. All of + this assembles the `Translation` the phase-1 plumbing already consumes. +- **Availability:** `available_languages(path)` = `sorted(node.langs)`; + hreflang alternates and `?lang=` handling use exactly this set. +- **Save (primary language):** server re-chunks the submitted Markdown, + inserts new hashes into `Data.chunks`, replaces `node.chunks`. Unchanged + chunks keep their hashes — only genuinely new text lands in the diff. +- **Save (translated view):** diff against the served hybrid, append a + `Patch` under `patches[f"{path}:{lang}"]`; `node.chunks` untouched. +- **Invalidate:** any write to `chunks` / `trans` / `patches` calls + `_invalidate_pages()`. + +## migrate_v3 steps + +1. Walk `menu`; for every node with a string `content`: + `chunks = chunk_markdown(content)`; write each into the new `chunks` + store; replace the field with the hash list (`None` stays `None`). +2. Initialize empty `chunks` / `trans` / `patches` stores. +3. Normalize stored paths: strip leading slashes anywhere paths are keys or + values. +4. `language`, `no_trans` and `langs` need nothing — struct defaults cover + them (`langs` starts empty; the translator job fills it as translations + land). + +Chunking must be deterministic and shared with render/save, so +`chunk_markdown` + `chunk_key` live in `pagerite/i18n.py` (or a small +`pagerite/chunks.py`) and are imported by both `migrations.py` and +`views.py`/`app.py`. + +## Garbage collection (later, manual or idle-time) + +Orphaned entries accumulate: chunks no longer referenced by any +`node.chunks`/`node.title`, translations whose chunk hash is orphaned, patch +hunks that never match. All are harmless (never read). A GC pass is a single +tree walk collecting live hashes, then deleting the rest from `chunks` and +`trans`; patches whose every hunk is stale get pruned. Not part of +migrate_v3. diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js index 1b9351b..5b95d6f 100644 --- a/frontend/src/pagerite.js +++ b/frontend/src/pagerite.js @@ -34,6 +34,24 @@ import "overlayscrollbars/overlayscrollbars.css"; }); } + // --- Language override (?lang=) --------------------------------------- + // /page?lang=fi serves a translated, indexable version (each language is + // its own canonical). Restore the pretty URL on load but remember the + // language: all fetch-navigation and preload requests below send it as + // Accept-Language, so the chosen language sticks for the session of + // clicks. A full refresh or shared link resets to automatic selection + // (the browser's own Accept-Language). See docs/localization.md. + const langParam = new URL(location.href).searchParams.get("lang"); + if (langParam) { + const url = new URL(location.href); + url.searchParams.delete("lang"); + history.replaceState(history.state, "", url); + } + const pageHeaders = langParam ? { "Accept-Language": langParam } : {}; + // The in-memory page cache is keyed per language: the same pathname holds + // different HTML for each selected language. + const cacheKey = (pathname) => (langParam ? `${langParam}|${pathname}` : pathname); + // Regions every page has. #sidebar is NOT among them: it is omitted // entirely when the section has no sub-navigation, and handled below. const REGIONS = ["page-banner", "nav", "main"]; @@ -352,9 +370,9 @@ import "overlayscrollbars/overlayscrollbars.css"; // received it as the document (re-fetching would be redundant, and // browser heuristics may send it without if-none-match, defeating the // conditional request); it enters the cache when navigated to. - const pageCache = new Map(); // pathname -> HTML text + const pageCache = new Map(); // cacheKey(pathname) -> HTML text addEventListener("pagerite:page-fetched", (ev) => { - pageCache.set(new URL(ev.detail.url, location.href).pathname, ev.detail.html); + pageCache.set(cacheKey(new URL(ev.detail.url, location.href).pathname), ev.detail.html); }); // Editors mutate site-wide state (theme, structure, headings, banners), @@ -380,14 +398,14 @@ import "overlayscrollbars/overlayscrollbars.css"; urls.add(a.pathname); } for (const url of urls) { - if (pageCache.has(url)) continue; + if (pageCache.has(cacheKey(url))) continue; // x-pagerite-preload: idle cache warm-up, not a page view — the // server excludes these GETs from analytics (the navigation message // sent on actual navigation does the counting). - fetch(url, { headers: { "x-pagerite-preload": "1" } }) + fetch(url, { headers: { "x-pagerite-preload": "1", ...pageHeaders } }) .then((r) => (r.ok && (r.headers.get("content-type") || "").includes("text/html") ? r.text() : "")) - .then((html) => { if (html) pageCache.set(url, html); }) + .then((html) => { if (html) pageCache.set(cacheKey(url), html); }) .catch(() => {}); } } @@ -697,12 +715,12 @@ import "overlayscrollbars/overlayscrollbars.css"; teardownAnalytics(); let doc; let finalUrl = url; - const cached = !editing && pageCache.get(new URL(url, location.href).pathname); + const cached = !editing && pageCache.get(cacheKey(new URL(url, location.href).pathname)); if (cached) { doc = new DOMParser().parseFromString(cached, "text/html"); } else { try { - const res = await fetch(url); + const res = await fetch(url, { headers: pageHeaders }); const type = res.headers.get("content-type") || ""; if (!res.ok || !type.includes("text/html")) throw new Error("not a page"); // Reflect any redirect the server issued. @@ -710,7 +728,7 @@ import "overlayscrollbars/overlayscrollbars.css"; const html = await res.text(); // Populate the cache too, so returning here (back/forward, or a // self-link in the nav) is served from memory. - pageCache.set(new URL(finalUrl, location.href).pathname, html); + pageCache.set(cacheKey(new URL(finalUrl, location.href).pathname), html); doc = new DOMParser().parseFromString(html, "text/html"); } catch { location.href = url; // fall back to a normal navigation diff --git a/pagerite/app.py b/pagerite/app.py index 0ef5d0b..0e6af55 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -47,7 +47,7 @@ from mediapreview import dispatch from pydantic import BaseModel from zstandard import ZstdCompressor -from pagerite import analytics, seed, views +from pagerite import analytics, i18n, seed, views from pagerite.__main__ import DEVMODE from pagerite.data import ( Data, @@ -381,10 +381,13 @@ class FileStore: file_store = FileStore(FILES_DIR) -def _render_html(kind: str, path: str, base_url: str) -> str: +def _render_html(kind: str, path: str, base_url: str, lang: str = i18n.ORIGINAL_LANGUAGE) -> str: """Render one of the generated pages (see _html_response).""" if kind == "page": - return views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition) + # A selected language without an actual translation renders the + # original (translation is None = English; see docs/localization.md). + translation = i18n.get_translation(path, lang) if lang != i18n.ORIGINAL_LANGUAGE else None + return views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition, lang=lang, translation=translation) if kind == "category": return views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) if kind == "not-found": @@ -406,13 +409,14 @@ def _invalidate_pages() -> None: @lru_cache(maxsize=128) -def _cached_body(kind: str, path: str, base_url: str, zstd: bool) -> bytes: +def _cached_body(kind: str, path: str, base_url: str, zstd: bool, lang: str = i18n.ORIGINAL_LANGUAGE) -> bytes: """Rendered page body; cleared by _invalidate_pages on any - content/settings change. base_url feeds the social meta URLs and zstd + content/settings change. base_url feeds the social meta URLs, zstd selects the stored encoding (both variants are cached rather than - re-compressed). + re-compressed) and lang the selected language (not the raw + Accept-Language header, which would blow up the cache key space). """ - body = _render_html(kind, path, base_url).encode() + body = _render_html(kind, path, base_url, lang).encode() return _zstd.compress(body) if zstd else body @@ -423,6 +427,7 @@ def _html_response( status_code: int = 200, headers: dict | None = None, etag: bool = False, + lang: str = i18n.ORIGINAL_LANGUAGE, ) -> Response: """Response for a generated page, zstd-compressed when the client accepts it (no gzip fallback). @@ -444,14 +449,15 @@ def _html_response( # localhost (varying ports) fall back to the request's own base URL. base_url = SITE_URL or str(request.base_url).rstrip("/") if DEVMODE: - identity = _render_html(kind, path, base_url).encode() + identity = _render_html(kind, path, base_url, lang).encode() body = _zstd.compress(identity) if zstd else identity else: - identity = _cached_body(kind, path, base_url, False) - body = _cached_body(kind, path, base_url, True) if zstd else identity + identity = _cached_body(kind, path, base_url, False, lang) + body = _cached_body(kind, path, base_url, True, lang) if zstd else identity h = dict(headers or {}) - if zstd: - h["vary"] = "accept-encoding" + # Content varies by language (Accept-Language selects a translation) + # and by encoding; keep caches from mixing either representation. + h["vary"] = "accept-language" + (", accept-encoding" if zstd else "") if etag: tag = f'"{blake3.blake3(identity).hexdigest()[:32]}"' h["etag"] = tag @@ -1509,13 +1515,22 @@ async def show_page(request: Request, path: str) -> Response: chain = resolve(data.menu, path) node = chain[-1] if chain else None if node is not None and node.published and node.content is not None: + # Language selection (docs/localization.md): ?lang= wins when a + # translation exists, else header logic. Analytics keep the raw + # Accept-Language header regardless of the selection. + languages = i18n.available_languages(path) + lang = i18n.select_language( + request.query_params.get("lang"), + accept_language, + lambda l: l in languages, + ) # no-cache forbids serving a stored page without revalidation # (browsers would otherwise cache heuristically and serve stale # pages, e.g. after a theme change). In-session speed instead comes # from pagerite.js's in-memory page cache (preload everything, never # fetch on navigation); the ETag just makes those one-time preload # fetches and any revalidation cheap. - etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}"' + etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}l{lang}"' if request.headers.get("if-none-match") == etag: return Response(status_code=304) if _is_trackable_path(path): @@ -1530,6 +1545,7 @@ async def show_page(request: Request, path: str) -> Response: "last-modified": _http_date(node.modified), "cache-control": "no-cache", }, + lang=lang, ) if node is not None and node.published and node.content is None: # Category label without a landing page: placeholder with the pen diff --git a/pagerite/i18n.py b/pagerite/i18n.py new file mode 100644 index 0000000..126d8dd --- /dev/null +++ b/pagerite/i18n.py @@ -0,0 +1,89 @@ +"""Localization: language selection and translation access. + +See docs/localization.md. The database keeps only the original language +(English); translations of page Markdown and navigation titles are consumed +through get_translation(), with per-node fallback to the original titles. +""" + +from collections.abc import Callable + +import msgspec + +#: Language of the database originals (and the default ). +ORIGINAL_LANGUAGE = "en" + + +class Translation(msgspec.Struct, omit_defaults=True): + """Translated content for one page and language. + + ``markdown`` is the translated page source in the same format as the + original (None = keep the original Markdown); ``titles`` maps node paths + (top-level slug, then slash-joined) to translated navigation titles, so a + partially translated tree still renders with per-node English fallback. + """ + + markdown: str | None = None + titles: dict[str, str] = {} + + +def parse_accept_language(header: str) -> list[str]: + """Accept-Language header as an ordered, deduped list of base subtags. + + q-values are deliberately ignored: all known implementations send the + header in order of preference. Region tags normalize to their base + subtag (fi-FI -> fi); "*" and empties are dropped. + """ + langs = [] + for part in header.split(","): + tag = part.split(";", 1)[0].strip().lower().partition("-")[0] + if tag and tag != "*" and tag not in langs: + langs.append(tag) + return langs + + +def select_language( + query_lang: str | None, + accept_language: str | None, + is_available: Callable[[str], bool], + original: str = ORIGINAL_LANGUAGE, +) -> str: + """The language to serve (see docs/localization.md). + + 1. ``?lang=`` wins when a translation exists for it (otherwise falls + through to the header logic). + 2. The original language anywhere in the header list wins — an AI + translation is strictly worse than the original for anyone who has + English configured at all. + 3. Otherwise the first header language with an available translation. + 4. Fall back to the original. + """ + if query_lang: + tag = query_lang.strip().lower().partition("-")[0] + if tag == original or (tag and is_available(tag)): + return tag + langs = parse_accept_language(accept_language or "") + if original in langs: + return original + for lang in langs: + if lang != original and is_available(lang): + return lang + return original + + +def get_translation(path: str, lang: str) -> Translation | None: + """The translation of the page at ``path`` for ``lang``, or None. + + TODO: stub for the AI-translation track, which will provide the real + implementation (storage, generation, invalidation on source changes). + The CMS only consumes this function and available_languages(). + """ + return None + + +def available_languages(path: str) -> list[str]: + """Languages with a translation for the page at ``path`` (besides the + original). Drives language selection and hreflang alternate links. + + TODO: stub, together with get_translation(). + """ + return [] diff --git a/pagerite/views.py b/pagerite/views.py index d61ee52..6cb986c 100644 --- a/pagerite/views.py +++ b/pagerite/views.py @@ -24,7 +24,9 @@ import re from html5tagger import HTML, Document, E, Template from platformdirs import site_data_dir, user_data_path +from pagerite import i18n from pagerite.data import Node, prettify, resolve, sorted_nodes +from pagerite.i18n import Translation from pagerite.markdown import render SITE_NAME = "Pagerite" @@ -309,6 +311,8 @@ def _layout( transition: str = "cube", favicon: str = "", social: dict[str, str] | None = None, + lang: str = i18n.ORIGINAL_LANGUAGE, + alternates: list[tuple[str, str]] = (), ) -> Template: """Page layout template with standard assets and ES-module scripts. @@ -333,8 +337,12 @@ def _layout( ``social`` maps meta keys to contents: ``og:*``/``article:*`` go out as property attributes, everything else (description, twitter:*) as name. + + ``lang`` is the served language for ; ``alternates`` holds + (hreflang, href) pairs for the other language versions of the page, + emitted as (see docs/localization.md). """ - doc = Document(E.Title, lang="en") + doc = Document(E.Title, lang=lang) # Responsive layout (see the 48rem breakpoint in pagerite.css) needs # the real device width, not the default 980px layout viewport. doc.meta(name="viewport", content="width=device-width, initial-scale=1") @@ -346,6 +354,8 @@ def _layout( doc.link(rel="canonical", href=value) else: doc.meta(name=key, content=value) + for hreflang, href in alternates: + doc.link(rel="alternate", hreflang=hreflang, href=href) # A custom favicon (from the site editor) is linked explicitly; without # one, browsers fall back to the build's /favicon.ico by convention. if favicon: @@ -452,14 +462,20 @@ def _brand_link(brand: str, brand_html: str = "") -> HTML: return HTML(str(E.a(brand, href="/", id="brand"))) if brand else HTML("") -def _title(slug: str, node: Node) -> str: - """Menu label: the configured title, prettified slug, "Home" fallback.""" +def _title(slug: str, node: Node, translation: Translation | None = None, path: str = "") -> str: + """Menu label: the configured title, prettified slug, "Home" fallback. + + With a translation, its title map (keyed by node path) wins, falling + back per node to the original English title. + """ + if translation and (t := translation.titles.get(path)): + return t return node.title or prettify(slug) or "Home" def _nav_link( doc, menu: dict[str, Node], node: Node, path: str, current: str, - ancestors_current: bool = True, + ancestors_current: bool = True, translation: Translation | None = None, ) -> None: """Render one
  • linking the node. Category labels (no content of their own — None, or empty markdown as left by the site editor's @@ -474,13 +490,13 @@ def _nav_link( if not node.content and (leaf := first_leaf(menu, path)) is not None: href = f"/{leaf}" doc.li.a( - _title(path.rpartition("/")[2], node), + _title(path.rpartition("/")[2], node, translation, path), href=href, **{"class": "current"} if is_current else {}, ) -def nav_html(menu: dict[str, Node], current: str) -> HTML: +def nav_html(menu: dict[str, Node], current: str, translation: Translation | None = None) -> HTML: """Render the contents of the #nav element for the current path. Top-level items in menu order; the front page (slug "", href "/") @@ -491,11 +507,11 @@ def nav_html(menu: dict[str, Node], current: str) -> HTML: with nav: for slug, node in sorted_nodes(menu): if node.published: - _nav_link(nav, menu, node, slug, current) + _nav_link(nav, menu, node, slug, current, translation=translation) return HTML(str(nav)) -def sidebar_html(menu: dict[str, Node], current: str) -> HTML: +def sidebar_html(menu: dict[str, Node], current: str, translation: Translation | None = None) -> HTML: """Render the #sidebar element for the current path (empty when none). The sidebar is the current main level section's sub-navigation: the @@ -529,20 +545,20 @@ def sidebar_html(menu: dict[str, Node], current: str) -> HTML: nav = E.ul with nav: for slug, child in items: - _sidebar_item(nav, menu, child, f"{section}/{slug}", current) + _sidebar_item(nav, menu, child, f"{section}/{slug}", current, translation) return HTML(str(E.aside(nav, id="sidebar"))) -def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: str) -> None: +def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: str, translation: Translation | None = None) -> None: """One sidebar
  • : the node link, with its published children as a nested list (third level and deeper, recursively).""" - _nav_link(doc, menu, node, path, current, ancestors_current=False) + _nav_link(doc, menu, node, path, current, ancestors_current=False, translation=translation) sub = [(s, c) for s, c in sorted_nodes(node.children) if c.published] if sub: # doc.li.a(...) above left the
  • open for nesting. with doc.ul: for slug, child in sub: - _sidebar_item(doc, menu, child, f"{path}/{slug}", current) + _sidebar_item(doc, menu, child, f"{path}/{slug}", current, translation) def first_leaf(menu: dict[str, Node], path: str) -> str | None: @@ -674,16 +690,24 @@ def banner_source(menu: dict[str, Node], path: str) -> str | None: return None -def page_content(menu: dict[str, Node], path: str) -> HTML: +def page_content(menu: dict[str, Node], path: str, translation: Translation | None = None) -> HTML: """Render the contents of the #main element for a page. A page with published children (a category page) lists them as cards - after the markdown content. + after the markdown content. With a translation, its Markdown goes + through the same render pipeline; missing pieces (markdown=None, absent + title entries) fall back to the original. """ node = resolve(menu, path)[-1] + content = node.content or "" + title = node.title + if translation: + if translation.markdown is not None: + content = translation.markdown + title = _title(path.rpartition("/")[2], node, translation, path) if node.title else title # The title is injected into the markdown (as # title when it has no # h1 of its own), so title and content render as one article. - rendered = render(node.content or "", path, node.created, node.modified, title=node.title) + rendered = render(content, path, node.created, node.modified, title=title) # Long articles get .multicol: the article column cap lifts (see the # #content grid in pagerite.css) and the .cols segments lay out in at # most two columns. The html is already segmented by render() — the @@ -691,11 +715,11 @@ def page_content(menu: dict[str, Node], path: str) -> HTML: doc = E.article(class_="multicol") if rendered.multicol else E.article with doc: doc(HTML(rendered.html)) - _cards(doc, menu, node, path) + _cards(doc, menu, node, path, translation) return HTML(str(doc)) -def _cards(doc, menu: dict[str, Node], node: Node, path: str) -> None: +def _cards(doc, menu: dict[str, Node], node: Node, path: str, translation: Translation | None = None) -> None: """Card stacks of the node's published children (nothing when childless). One column per direct child, all in a single full-width row (the .wide @@ -721,7 +745,7 @@ def _cards(doc, menu: dict[str, Node], node: Node, path: str) -> None: continue with doc.div(class_="stack"): for epath, enode in entries: - _card(doc, enode, epath) + _card(doc, enode, epath, translation) def _walk(node: Node, path: str): @@ -735,9 +759,13 @@ def _walk(node: Node, path: str): yield from _walk(child, f"{path}/{slug}") -def _card(doc, node: Node, path: str) -> None: +def _card(doc, node: Node, path: str, translation: Translation | None = None) -> None: """One card in a stack: cover + title, plus the description when the - page has no image (its card shows a gradient cover instead).""" + page has no image (its card shows a gradient cover instead). + + Cover/description heuristics run on the original Markdown even when + translated (only this page's own Markdown is translated); the title + uses the translation's title map.""" image = description = "" if node.content: html = render(node.content, path, node.created, node.modified).html @@ -749,7 +777,7 @@ def _card(doc, node: Node, path: str) -> None: doc.span(class_="cover", style=f'background-image: url("{image}")') else: doc.span(class_="cover") - doc.span(_title(path.rpartition("/")[2], node), class_="title") + doc.span(_title(path.rpartition("/")[2], node, translation, path), class_="title") if description: doc.span(description, class_="desc") @@ -833,6 +861,7 @@ def _share_media(html: str, base_url: str) -> tuple[str, str]: def _social_meta( node: Node, path: str, title: str, html: str, brand: str, base_url: str, + lang: str = i18n.ORIGINAL_LANGUAGE, ) -> dict[str, str]: """Open Graph/Twitter/SEO meta tags for a content page. @@ -842,6 +871,9 @@ def _social_meta( representative figure. Absolute URLs are built from the request's base (social scrapers cannot use relative ones). + When a translation is served, the canonical URL includes the ?lang= + query: each language version is its own canonical (docs/localization.md). + ``twitter:image`` pins extension-less store links to the ``.webp`` variant: X only honors WebP via twitter:image (not og:image) and its scraper cannot be trusted to negotiate via Accept. @@ -854,7 +886,7 @@ def _social_meta( ) return { "description": text, - "canonical": url, + "canonical": f"{url}?lang={lang}" if url and lang != i18n.ORIGINAL_LANGUAGE else url, "og:type": "article", "og:title": title, "og:description": text, @@ -879,21 +911,39 @@ def render_page( brand_html: str = "", base_url: str = "", transition: str = "cube", + lang: str = i18n.ORIGINAL_LANGUAGE, + translation: Translation | None = None, ) -> str: - """Render a full HTML page for the slug path.""" + """Render a full HTML page for the slug path. + + ``lang``/``translation`` serve a translated version (see + docs/localization.md): None translation = the English original. + """ node = resolve(menu, path)[-1] - title = _title(path.rpartition("/")[2], node) - main = page_content(menu, path) - social = _social_meta(node, path, title, str(main), brand, base_url) + if translation is None: + lang = i18n.ORIGINAL_LANGUAGE + title = _title(path.rpartition("/")[2], node, translation, path) + main = page_content(menu, path, translation) + social = _social_meta(node, path, title, str(main), brand, base_url, lang) + # hreflang alternates: every other language version (with ?lang=) plus + # x-default for the plain URL. Emitted only when translations exist. + alternates = [] + if base_url: + available = i18n.available_languages(path) + alternates = [ + (l, f"{base_url}/{path}?lang={l}") for l in available if l != lang + ] + if available: + alternates.append(("x-default", f"{base_url}/{path}")) return str( _layout( *_page_assets(), custom_css, theme, banner_design(menu, path, theme), - transition, favicon, social, + transition, favicon, social, lang, alternates, )( Title=f"{title} – {brand}" if brand else title, Brand=_brand_link(brand, brand_html), - Nav=nav_html(menu, path), - Sidebar=sidebar_html(menu, path), + Nav=nav_html(menu, path, translation), + Sidebar=sidebar_html(menu, path, translation), Banner=banner_html(menu, path, theme), Main=main, ), -- 2.55.0 From 8558345edafcf07edf60ccca6d0afb2dae14a086 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:29:10 +0000 Subject: [PATCH 02/31] Add chunks.py: deterministic block-level Markdown chunking + content keys --- pagerite/chunks.py | 150 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 pagerite/chunks.py diff --git a/pagerite/chunks.py b/pagerite/chunks.py new file mode 100644 index 0000000..9d118e6 --- /dev/null +++ b/pagerite/chunks.py @@ -0,0 +1,150 @@ +"""Block-level Markdown chunking for content-addressed storage. + +A page's Markdown is split into deterministic block-level chunks, each +stored once under its content hash in ``Data.chunks`` (docs/migrate.md). +Shared by the render/save pipeline (app.py, views.py, i18n.py) and the +schema migration (migrations.py), so a chunk's key is stable no matter +where the split happens. +""" + +import re + +import blake3 + +#: Fenced code block opener/closer: up to 3 spaces indent, then 3+ +#: backticks or tildes (CommonMark). +_FENCE_OPEN = re.compile(r"^ {0,3}(`{3,}|~{3,})") + +#: HTML block openers that may span blank lines (CommonMark types 1-5: +#: script/pre/style/textarea, comments, processing instructions, +#: declarations, CDATA) with their closing condition. Other HTML blocks +#: end at the first blank line, which the generic blank-line split +#: already does. +_HTML_ATOMIC = ( + (re.compile(r"^ {0,3}<(?:script|pre|style|textarea)(?:\s|>|$)", re.I), + re.compile(r"", re.I)), + (re.compile(r"^ {0,3}")), + (re.compile(r"^ {0,3}<\?"), re.compile(r"\?>")), + (re.compile(r"^ {0,3}")), + (re.compile(r"^ {0,3}")), +) + +#: First line of a generic HTML block (a block-level tag). +_HTML_TAG = re.compile(r"^ {0,3}]*>") + + +def _fence_close(line: str, opener: str) -> bool: + """True when ``line`` closes a code fence opened by ``opener``: the + same marker char, at least as many, and nothing else on the line.""" + stripped = line.strip() + return ( + len(stripped) >= len(opener) + and stripped[0] == opener[0] + and set(stripped) == {opener[0]} + ) + + +def chunk_markdown(markdown: str) -> list[str]: + """Split Markdown into block-level chunks, deterministically. + + Blocks are separated by blank lines; fenced code blocks and the + multi-line HTML blocks (comments, script/pre/style, CDATA...) are + kept atomic, even across blank lines, and end at their closing + condition. Chunks carry no surrounding blank lines and no trailing + newline; rejoining with ``join_chunks`` reproduces the source modulo + blank-line normalization. + """ + chunks: list[str] = [] + buf: list[str] = [] + fence = "" # opener marker of the code fence we are in ("" = outside) + html_end: re.Pattern | None = None # closes the atomic HTML block we are in + + def flush() -> None: + text = "\n".join(buf).strip("\n") + if text.strip(): + chunks.append(text) + buf.clear() + + for line in markdown.split("\n"): + if fence: + buf.append(line) + if _fence_close(line, fence): + fence = "" + flush() + continue + if html_end is not None: + buf.append(line) + if html_end.search(line): + html_end = None + flush() + continue + if not line.strip(): + flush() + continue + if m := _FENCE_OPEN.match(line): + # Fences interrupt paragraphs (CommonMark): start a new block. + flush() + fence = m.group(1) + buf.append(line) + continue + if not buf: + for open_re, close_re in _HTML_ATOMIC: + if open_re.match(line): + buf.append(line) + if close_re.search(line): # opens and closes on one line + flush() + else: + html_end = close_re + break + else: + buf.append(line) + continue + buf.append(line) + flush() # an unterminated fence/HTML block runs to EOF, kept as code/HTML + return chunks + + +def _normalize(text: str) -> str: + """Whitespace-insensitive chunk identity: strip trailing whitespace + per line and collapse surrounding blank lines, so whitespace-only + source edits don't invalidate translations.""" + return "\n".join(line.rstrip() for line in text.split("\n")).strip("\n") + + +def chunk_key(text: str) -> str: + """Content key of a chunk: blake3 hex (16-byte digest, 32 hex chars) + of the normalized text — the same hasher app.py's file store uses.""" + return blake3.blake3(_normalize(text).encode()).hexdigest(16) + + +def needs_translation(chunk: str) -> bool: + """False for chunks without prose: pure code fences and HTML blocks. + + These are inherently no-translate (docs/migrate.md): derived from the + chunk text itself, nothing is stored. + """ + if _FENCE_OPEN.match(chunk): + return False + first = chunk.split("\n", 1)[0] + if any(open_re.match(first) for open_re, _ in _HTML_ATOMIC): + return False + return not _HTML_TAG.match(first) + + +def join_chunks(chunks: list[str]) -> str: + """The stored page form of chunks: blocks joined by a blank line, + with a trailing newline ("" for no chunks).""" + return "\n\n".join(chunks) + "\n" if chunks else "" + + +def store_chunks(store: dict[str, str], markdown: str) -> list[str]: + """Chunk ``markdown`` into ``store`` (hash -> text); return the ordered + hashes. Unchanged chunks keep their hashes, so only genuinely new text + lands in the kanta change diff. First writer wins: variants sharing a + key differ only in insignificant whitespace (see chunk_key).""" + hashes = [] + for chunk in chunk_markdown(markdown): + key = chunk_key(chunk) + store.setdefault(key, chunk) + hashes.append(key) + return hashes -- 2.55.0 From 9bc71b87171a5c1e71c13677fda38dc9a0600079 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:30:04 +0000 Subject: [PATCH 03/31] Data model v3: content-addressed chunk stores, Patch, Node.chunks/langs --- pagerite/data.py | 55 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/pagerite/data.py b/pagerite/data.py index 9a105fc..056b4ad 100644 --- a/pagerite/data.py +++ b/pagerite/data.py @@ -2,8 +2,9 @@ The site structure is a tree of Nodes. Every node is a menu label with a configurable title and slug (its key in the parent's ``children``); the -URL path is the chain of slugs from the top level. ``content`` is the -node's Markdown page, or None for a pure category label, whose URL renders +URL path is the chain of slugs from the top level. ``chunks`` is the +node's Markdown page as ordered content-hash keys into ``Data.chunks`` +(docs/migrate.md), or None for a pure category label, whose URL renders a placeholder page while nav links point at its first child. """ @@ -11,6 +12,16 @@ from datetime import UTC, datetime import msgspec +from pagerite.chunks import join_chunks + + +class Patch(msgspec.Struct, omit_defaults=True): + """One editing session's overrides on a translated view, applied + independently per hunk (docs/localization.md).""" + + #: (search, replace) pairs on the served hybrid Markdown. + hunks: list[tuple[str, str]] = [] + class Node(msgspec.Struct, omit_defaults=True): """One item of the site hierarchy. @@ -28,9 +39,21 @@ class Node(msgspec.Struct, omit_defaults=True): title: str = "" order: float = 0 - #: Markdown source of the node's page; None = pure category label - #: (its URL renders a placeholder page). - content: str | None = None + #: Ordered chunk hashes into ``Data.chunks``; None = pure category + #: label (its URL renders a placeholder page), a list (possibly + #: empty) = a page. + chunks: list[str] | None = None + #: Primary language of the article (BCP-47 base tag). "" = inherit + #: (nearest ancestor, front page last, site default "en" final). + language: str = "" + #: Chunk hashes the editor marked "do not translate" (always served + #: from the original). Presence-keys, value always True. + no_trans: dict[str, bool] = {} + #: Languages this article is available in (besides its primary + #: language). Presence-keys, value always True — the availability + #: index for rendering, language selection and hreflang alternates; + #: maintained by whoever writes translation data (docs/migrate.md). + langs: dict[str, bool] = {} #: Raw HTML for the header banner (img, styled div, canvas+script...), #: rendered after the banner design's artwork so author code always #: wins over the design's own styles. @@ -78,6 +101,28 @@ class Data(msgspec.Struct): #: linked as on every page. Empty = the build's #: /favicon.ico. favicon: str = "" + #: All original-language page text, content-addressed: + #: chunk_key -> Markdown chunk. Shared by every article. + chunks: dict[str, str] = {} + #: Machine translations: f"{chunk_hash}:{lang}" -> translated + #: Markdown. Also used for node titles (hash of the title text). + trans: dict[str, str] = {} + #: User override patches per article and language: + #: f"{path}:{lang}" -> ordered patches (paths without leading slash). + patches: dict[str, list[Patch]] = {} + + +def node_markdown(data: Data, node: Node) -> str | None: + """The node's original Markdown assembled from the chunk store. + + None for category labels (chunks is None); an empty page gives "". + Hashes missing from the store (shouldn't happen) are skipped. + """ + if node.chunks is None: + return None + return join_chunks( + [t for h in node.chunks if (t := data.chunks.get(h)) is not None] + ) def prettify(slug: str) -> str: -- 2.55.0 From c4e731b3d00b4869f22c2a9de6903c2015c370c5 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:31:29 +0000 Subject: [PATCH 04/31] migrate_v3: split node content into content-addressed chunks --- pagerite/migrations.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pagerite/migrations.py b/pagerite/migrations.py index d37f10a..ed61ad2 100644 --- a/pagerite/migrations.py +++ b/pagerite/migrations.py @@ -15,6 +15,7 @@ import base64 import re from pathlib import Path +from pagerite.chunks import chunk_key, chunk_markdown from pagerite.data import prettify @@ -131,3 +132,38 @@ def migrate_v2(d: dict) -> None: walk(d.get("menu") or {}) d.pop("version", None) _backfill_derivatives() + + +def migrate_v3(d: dict) -> None: + """Content-addressed chunk storage (docs/migrate.md): split every + node's string ``content`` into block chunks stored once per content + hash in the new ``chunks`` store; the node keeps the ordered hash + list as ``chunks`` (an absent content stays absent, i.e. None = a + pure category label; "" chunks to an empty list = an empty page). + + ``trans``/``patches`` start empty; the translator job fills them and + maintains the ``langs`` index as translations land. ``language``, + ``no_trans`` and ``langs`` need nothing — struct defaults cover them. + """ + store = d.setdefault("chunks", {}) + d.setdefault("trans", {}) + patches = d.setdefault("patches", {}) + + def walk(nodes: dict) -> None: + for node in nodes.values(): + content = node.pop("content", None) + if isinstance(content, str): + hashes = [] + for chunk in chunk_markdown(content): + key = chunk_key(chunk) + store.setdefault(key, chunk) + hashes.append(key) + node["chunks"] = hashes + walk(node.get("children") or {}) + + walk(d.get("menu") or {}) + # Article paths never carry a leading slash in keys (docs/migrate.md). + # The only path-keyed store starts empty here, so this is defensive + # for databases that went through a downgrade/upgrade cycle. + for key in [k for k in patches if k.startswith("/")]: + patches[key.lstrip("/")] = patches.pop(key) -- 2.55.0 From 6e9f73f83a547124b7d37cc82bdd1af251dabd46 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:34:13 +0000 Subject: [PATCH 05/31] Real translation storage: chunk lookup, user patches, title map --- pagerite/i18n.py | 136 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 18 deletions(-) diff --git a/pagerite/i18n.py b/pagerite/i18n.py index 126d8dd..6ae7802 100644 --- a/pagerite/i18n.py +++ b/pagerite/i18n.py @@ -1,14 +1,20 @@ -"""Localization: language selection and translation access. +"""Localization: language selection, translation storage and assembly. -See docs/localization.md. The database keeps only the original language -(English); translations of page Markdown and navigation titles are consumed -through get_translation(), with per-node fallback to the original titles. +See docs/localization.md and docs/migrate.md. The database holds the +original language as content-addressed chunks (``Data.chunks``); per +target language there are machine-translated fragments (``Data.trans``) +and user override patches (``Data.patches``), assembled into the served +Markdown at render time, with per-node fallback to the original titles. """ from collections.abc import Callable +from difflib import SequenceMatcher import msgspec +from pagerite.chunks import chunk_key, chunk_markdown, join_chunks +from pagerite.data import Data, Node, Patch, resolve + #: Language of the database originals (and the default ). ORIGINAL_LANGUAGE = "en" @@ -26,6 +32,11 @@ class Translation(msgspec.Struct, omit_defaults=True): titles: dict[str, str] = {} +def base_tag(tag: str) -> str: + """The lowercase base subtag of a language tag (fi-FI -> fi).""" + return tag.strip().lower().partition("-")[0] + + def parse_accept_language(header: str) -> list[str]: """Accept-Language header as an ordered, deduped list of base subtags. @@ -35,7 +46,7 @@ def parse_accept_language(header: str) -> list[str]: """ langs = [] for part in header.split(","): - tag = part.split(";", 1)[0].strip().lower().partition("-")[0] + tag = base_tag(part.split(";", 1)[0]) if tag and tag != "*" and tag not in langs: langs.append(tag) return langs @@ -58,7 +69,7 @@ def select_language( 4. Fall back to the original. """ if query_lang: - tag = query_lang.strip().lower().partition("-")[0] + tag = base_tag(query_lang) if tag == original or (tag and is_available(tag)): return tag langs = parse_accept_language(accept_language or "") @@ -70,20 +81,109 @@ def select_language( return original -def get_translation(path: str, lang: str) -> Translation | None: +def apply_patch(hybrid: str, patch: Patch) -> str: + """Apply one patch to the hybrid Markdown, best effort, each hunk + independently: a hunk whose search text no longer exists is stale and + silently skipped (docs/localization.md).""" + for search, replace in patch.hunks: + if search and search in hybrid: + hybrid = hybrid.replace(search, replace, 1) + return hybrid + + +def make_patch(base: str, edited: str) -> Patch: + """The minimal diff of ``edited`` against the served ``base`` hybrid as + (search, replace) hunks at block granularity (docs/localization.md). + + Blocks are the chunk_markdown split, so hunks align with translation + units and code fences never straddle a hunk boundary. Pure inserts + anchor on the preceding block (an empty search would never match); + inserts at the very top anchor on the first block. autojunk is off: + the diff must be deterministic, and pages are small. + """ + a, b = chunk_markdown(base), chunk_markdown(edited) + hunks: list[tuple[str, str]] = [] + for tag, i1, i2, j1, j2 in SequenceMatcher(None, a, b, autojunk=False).get_opcodes(): + if tag == "equal": + continue + search = "\n\n".join(a[i1:i2]) + replace = "\n\n".join(b[j1:j2]) + if tag == "insert": + if i1: + search = a[i1 - 1] + replace = f"{a[i1 - 1]}\n\n{replace}" + elif a: + search = a[0] + replace = f"{replace}\n\n{a[0]}" + # else: base is empty — the hunk is inert (empty search is + # skipped by apply_patch); saving a translation of an empty + # page records nothing applicable. + hunks.append((search, replace)) + return Patch(hunks=hunks) + + +def hybrid_markdown(data: Data, node: Node, path: str, lang: str) -> str: + """The served Markdown for ``lang``: per chunk the translation from + ``Data.trans``, unless missing or marked no-translate (fallback to the + original chunk), then the language's user patches applied in order. + + Not gated on ``node.langs`` (get_translation is the gated view): the + editor save path diffs against this even for a language's first patch. + """ + hybrid = join_chunks([ + data.chunks.get(h, "") + if h in node.no_trans + else data.trans.get(f"{h}:{lang}") or data.chunks.get(h, "") + for h in node.chunks or [] + ]) + for patch in data.patches.get(f"{path}:{lang}", []): + hybrid = apply_patch(hybrid, patch) + return hybrid + + +def title_map(data: Data, lang: str) -> dict[str, str]: + """path -> translated title for every node that has one. + + Titles are chunks too (docs/migrate.md): keyed by the hash of the + title text, so editing a title invalidates its translations. Nodes + without an entry fall back to their original title in views. + """ + titles = {} + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in nodes.items(): + path = f"{prefix}/{slug}" if prefix else slug + if node.title: + t = data.trans.get(f"{chunk_key(node.title)}:{lang}") + if t: + titles[path] = t + walk(node.children, path) + + walk(data.menu, "") + return titles + + +def get_translation(path: str, lang: str, data: Data) -> Translation | None: """The translation of the page at ``path`` for ``lang``, or None. - TODO: stub for the AI-translation track, which will provide the real - implementation (storage, generation, invalidation on source changes). - The CMS only consumes this function and available_languages(). + None when the page does not exist or is not available in ``lang``: + ``node.langs`` is the availability index (a stale key is benign — the + "translation" then just renders as the original). """ - return None + chain = resolve(data.menu, path) + node = chain[-1] if chain else None + if node is None or node.chunks is None or lang not in node.langs: + return None + return Translation( + markdown=hybrid_markdown(data, node, path, lang), + titles=title_map(data, lang), + ) -def available_languages(path: str) -> list[str]: - """Languages with a translation for the page at ``path`` (besides the - original). Drives language selection and hreflang alternate links. - - TODO: stub, together with get_translation(). - """ - return [] +def available_languages(path: str, data: Data) -> list[str]: + """Languages the page at ``path`` is available in (besides the + original): the ``node.langs`` index, maintained by the translation + writers. Drives language selection and hreflang alternate links.""" + chain = resolve(data.menu, path) + node = chain[-1] if chain else None + return sorted(node.langs) if node else [] -- 2.55.0 From 98a1dc8d0f66a35de87ffa79557c3ed86e98f995 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:46:27 +0000 Subject: [PATCH 06/31] Rewire pages to chunk storage; translated-view open/save as patches --- pagerite/app.py | 143 +++++++++++++++++++++++++++++++++------------- pagerite/views.py | 36 ++++++------ 2 files changed, 122 insertions(+), 57 deletions(-) diff --git a/pagerite/app.py b/pagerite/app.py index 0e6af55..31155f0 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -49,11 +49,13 @@ from zstandard import ZstdCompressor from pagerite import analytics, i18n, seed, views from pagerite.__main__ import DEVMODE +from pagerite.chunks import store_chunks from pagerite.data import ( Data, Node, append_order, find_slot, + node_markdown, prettify, resolve, sorted_nodes, @@ -255,7 +257,7 @@ def _remove_page_content(menu: dict[str, Node], path: str) -> None: if node is None: return if node.children: - node.content = None + node.chunks = None node.modified = datetime.now(UTC) else: del slot[0][slot[1]] @@ -271,10 +273,10 @@ def _seed(data: Data) -> None: node = _ensure(data.menu, path) node.title = title # Empty markdown means a pure category label (e.g. "showcase", - # seeded only to carry a banner design): leave content as None so + # seeded only to carry a banner design): leave chunks as None so # the node renders the placeholder and nav points at its children. if markdown: - node.content = markdown + node.chunks = store_chunks(data.chunks, markdown) node.banner = banner node.banner_design = design node.order = order @@ -386,10 +388,10 @@ def _render_html(kind: str, path: str, base_url: str, lang: str = i18n.ORIGINAL_ if kind == "page": # A selected language without an actual translation renders the # original (translation is None = English; see docs/localization.md). - translation = i18n.get_translation(path, lang) if lang != i18n.ORIGINAL_LANGUAGE else None - return views.render_page(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition, lang=lang, translation=translation) + translation = i18n.get_translation(path, lang, data) if lang != i18n.ORIGINAL_LANGUAGE else None + return views.render_page(data.menu, data, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition, lang=lang, translation=translation) if kind == "category": - return views.render_category(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) + return views.render_category(data.menu, data, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) if kind == "not-found": return views.render_not_found(data.menu, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) return views.render_analytics(data.menu, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) @@ -494,7 +496,7 @@ async def list_pages() -> list[dict]: "title": node.title, "order": node.order, "published": node.published, - "has_content": node.content is not None, + "has_content": node.chunks is not None, "children": dump(node.children, path), }) return out @@ -503,7 +505,7 @@ async def list_pages() -> list[dict]: @app.put("/_api/pages/{path:path}", status_code=204) -async def save_page(path: str, page: PageIn) -> None: +async def save_page(path: str, page: PageIn, lang: str | None = None) -> None: """Create or replace the page at a slug path ("" or "/" = front page). Missing ancestors are created as content-less category labels. Giving @@ -511,13 +513,36 @@ async def save_page(path: str, page: PageIn) -> None: stripping) creates an empty page that renders with just its title — saving never deletes; use DELETE to remove a page (the page editor issues DELETE when you save empty text). + + With a ``?lang=`` query (a translation, not the primary language) the + save is a translated-view edit (docs/localization.md): the markdown is + diffed against the currently served hybrid and the minimal diff is + appended as a Patch under ``patches[f"{path}:{lang}"]`` — node.chunks + and the original-language fields (title, published, banner) stay + untouched. """ path = path.strip("/") _check_reserved(path) + lang = i18n.base_tag(lang or "") + if lang and lang != i18n.ORIGINAL_LANGUAGE: + chain = resolve(data.menu, path) + node = chain[-1] if chain else None + if node is None or node.chunks is None: + raise HTTPException(404, "no such page") + patch = i18n.make_patch( + i18n.hybrid_markdown(data, node, path, lang), page.markdown + ) + if patch.hunks: + with kanta.transaction("save translation", extra=path): + # Patches alone make the translated version exist. + data.patches.setdefault(f"{path}:{lang}", []).append(patch) + node.langs[lang] = True + _invalidate_pages() + return with kanta.transaction("save page", extra=path): node = _ensure(data.menu, path) node.title = page.title - node.content = page.markdown + node.chunks = store_chunks(data.chunks, page.markdown) node.published = page.published if page.banner is not None: node.banner = page.banner @@ -695,13 +720,15 @@ async def toggle_task_endpoint(body: ToggleTaskIn) -> dict[str, str]: return {"markdown": new_markdown} chain = resolve(data.menu, path) node = chain[-1] if chain else None - if node is None or node.content is None: + if node is None or node.chunks is None: raise HTTPException(404, "no such page") - new_markdown = toggle_task(node.content, body.index) + new_markdown = toggle_task(node_markdown(data, node) or "", body.index) if new_markdown is None: raise HTTPException(400, "invalid task index") with kanta.transaction("toggle task", extra=path): - node.content = new_markdown + # Re-chunk like any save: only the chunk containing the toggled + # checkbox gets a new hash, the rest keep theirs. + node.chunks = store_chunks(data.chunks, new_markdown) node.modified = datetime.now(UTC) _invalidate_pages() return {"markdown": new_markdown} @@ -931,7 +958,7 @@ async def delete_page(path: str) -> None: raise HTTPException(404, "no such page") with kanta.transaction("delete page", extra=path): if node.children: - node.content = None + node.chunks = None node.modified = datetime.now(UTC) else: del slot[0][slot[1]] @@ -1254,15 +1281,21 @@ async def editor_ws(ws: WebSocket) -> None: """Editor session: open pages, render previews, save — over one socket. Stateless protocol (each message carries the path): - <- {"type": "open", "path"} + <- {"type": "open", "path", "lang"?} -> {"type": "doc", "path", "exists", "title", "markdown", "published", "banner", "banner_design"} <- {"type": "render", "path", "markdown"} -> {"type": "html", "path", "html"} <- {"type": "save", "path", "title"?, "markdown"?, "published"?, - "banner"?, "banner_design"?, "move_from"?} (absent fields keep - their old values; move_from: rename/move a page, subtree included) + "banner"?, "banner_design"?, "move_from"?, "lang"?} (absent fields + keep their old values; move_from: rename/move a page, subtree + included) -> {"type": "saved", "path"} | {"type": "error", "detail"} + + With "lang" (a translation, not the primary language), open returns the + served hybrid Markdown for that language and save stores a diff against + it as a user Patch — node.chunks and the other fields stay untouched + (docs/localization.md). """ await ws.accept() try: @@ -1278,12 +1311,20 @@ async def editor_ws(ws: WebSocket) -> None: case "open": chain = resolve(data.menu, path) node = chain[-1] if chain else None + markdown = "" + if node is not None: + markdown = node_markdown(data, node) or "" + # ?lang= view: the served hybrid, not the raw + # original (docs/localization.md editor flow). + lang = i18n.base_tag(str(msg.get("lang") or "")) + if lang and (t := i18n.get_translation(path, lang, data)) is not None: + markdown = t.markdown or markdown await ws.send_json({ "type": "doc", "path": path, "exists": node is not None, "title": node.title if node else "", - "markdown": node.content if node and node.content is not None else "", + "markdown": markdown, "published": node.published if node else True, "banner": node.banner if node else "", # Own banner design setting: null = inherit, @@ -1331,6 +1372,8 @@ async def editor_ws(ws: WebSocket) -> None: }) case "save": move_from = (msg.get("move_from") or path).strip("/") + lang = i18n.base_tag(str(msg.get("lang") or "")) + translated = bool(lang and lang != i18n.ORIGINAL_LANGUAGE) try: _check_reserved(move_from) except HTTPException: @@ -1364,6 +1407,11 @@ async def editor_ws(ws: WebSocket) -> None: "detail": "target path exists", }) continue + if translated and (move_from != path or old is None or old.chunks is None): + # A translated-view save patches an existing + # original; it cannot create or move pages. + await ws.send_json({"type": "error", "detail": "no such page"}) + continue with kanta.transaction("editor save", extra=path): if move_from != path: same_menu = ( @@ -1381,21 +1429,37 @@ async def editor_ws(ws: WebSocket) -> None: tnodes[tslug] = node else: node = old if old is not None else _ensure(data.menu, path) - if "markdown" in msg: - # Saving never deletes; empty markdown is an - # empty page. Deletion is an explicit choice by - # the page editor (REST DELETE). - node.content = msg["markdown"] - if "title" in msg: - node.title = msg["title"] - if "published" in msg: - node.published = bool(msg["published"]) - if "banner" in msg: - node.banner = msg["banner"] - if "banner_design" in msg: - node.banner_design = msg["banner_design"] - node.modified = datetime.now(UTC) - _invalidate_pages() + if translated: + # Diff against the currently served hybrid and + # append a Patch; node.chunks and the + # original-language fields stay untouched. + if "markdown" in msg: + patch = i18n.make_patch( + i18n.hybrid_markdown(data, node, path, lang), + msg["markdown"], + ) + if patch.hunks: + # Patches alone make the translated + # version exist. + data.patches.setdefault(f"{path}:{lang}", []).append(patch) + node.langs[lang] = True + _invalidate_pages() + else: + if "markdown" in msg: + # Saving never deletes; empty markdown is an + # empty page. Deletion is an explicit choice + # by the page editor (REST DELETE). + node.chunks = store_chunks(data.chunks, msg["markdown"]) + if "title" in msg: + node.title = msg["title"] + if "published" in msg: + node.published = bool(msg["published"]) + if "banner" in msg: + node.banner = msg["banner"] + if "banner_design" in msg: + node.banner_design = msg["banner_design"] + node.modified = datetime.now(UTC) + _invalidate_pages() await ws.send_json({"type": "saved", "path": path}) except WebSocketDisconnect: pass @@ -1420,7 +1484,7 @@ async def sitemap(request: Request) -> Response: ( slug for slug, node in sorted_nodes(nodes) - if node.published and node.content is not None + if node.published and node.chunks is not None ), None, ) @@ -1431,14 +1495,14 @@ async def sitemap(request: Request) -> Response: not parent_has_content and slug == first_content_slug and node.published - and node.content is not None + and node.chunks is not None and depth > 0 ): depth -= 1 - if node.published and node.content is not None: + if node.published and node.chunks is not None: entries.append((path, node.modified, depth)) if node.children: - walk(node.children, path, node.content is not None) + walk(node.children, path, node.chunks is not None) walk(data.menu, "") @@ -1514,15 +1578,14 @@ async def show_page(request: Request, path: str) -> Response: raise HTTPException(404) chain = resolve(data.menu, path) node = chain[-1] if chain else None - if node is not None and node.published and node.content is not None: + if node is not None and node.published and node.chunks is not None: # Language selection (docs/localization.md): ?lang= wins when a # translation exists, else header logic. Analytics keep the raw # Accept-Language header regardless of the selection. - languages = i18n.available_languages(path) lang = i18n.select_language( request.query_params.get("lang"), accept_language, - lambda l: l in languages, + lambda l: l in node.langs, ) # no-cache forbids serving a stored page without revalidation # (browsers would otherwise cache heuristically and serve stale @@ -1547,7 +1610,7 @@ async def show_page(request: Request, path: str) -> Response: }, lang=lang, ) - if node is not None and node.published and node.content is None: + if node is not None and node.published and node.chunks is None: # Category label without a landing page: placeholder with the pen # to create it (404 — no page here, but the node is real). if _is_trackable_path(path): diff --git a/pagerite/views.py b/pagerite/views.py index 6cb986c..d4f4af1 100644 --- a/pagerite/views.py +++ b/pagerite/views.py @@ -25,7 +25,7 @@ from html5tagger import HTML, Document, E, Template from platformdirs import site_data_dir, user_data_path from pagerite import i18n -from pagerite.data import Node, prettify, resolve, sorted_nodes +from pagerite.data import Data, Node, node_markdown, prettify, resolve, sorted_nodes from pagerite.i18n import Translation from pagerite.markdown import render @@ -478,7 +478,7 @@ def _nav_link( ancestors_current: bool = True, translation: Translation | None = None, ) -> None: """Render one
  • linking the node. Category labels (no content of - their own — None, or empty markdown as left by the site editor's + their own — chunks None, or an empty page as left by the site editor's page creation) link straight to their first child page, so normal navigation bypasses the placeholder/empty page at their own URL.""" # The navbar highlights a top-level item also when viewing any of its @@ -487,7 +487,7 @@ def _nav_link( ancestors_current and path and current.startswith(f"{path}/") ) href = f"/{path}" - if not node.content and (leaf := first_leaf(menu, path)) is not None: + if not node.chunks and (leaf := first_leaf(menu, path)) is not None: href = f"/{leaf}" doc.li.a( _title(path.rpartition("/")[2], node, translation, path), @@ -562,7 +562,7 @@ def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: st def first_leaf(menu: dict[str, Node], path: str) -> str | None: - """First published descendant page (content set) in menu order. + """First published descendant page (chunks set) in menu order. This is the nav-link target for content-less category labels. """ @@ -577,7 +577,7 @@ def _first_leaf(node: Node, path: str) -> str | None: if not child.published: continue cpath = f"{path}/{slug}" if path else slug - if child.content: + if child.chunks: return cpath if (leaf := _first_leaf(child, cpath)) is not None: return leaf @@ -690,7 +690,7 @@ def banner_source(menu: dict[str, Node], path: str) -> str | None: return None -def page_content(menu: dict[str, Node], path: str, translation: Translation | None = None) -> HTML: +def page_content(menu: dict[str, Node], data: Data, path: str, translation: Translation | None = None) -> HTML: """Render the contents of the #main element for a page. A page with published children (a category page) lists them as cards @@ -699,7 +699,7 @@ def page_content(menu: dict[str, Node], path: str, translation: Translation | No title entries) fall back to the original. """ node = resolve(menu, path)[-1] - content = node.content or "" + content = node_markdown(data, node) or "" title = node.title if translation: if translation.markdown is not None: @@ -715,11 +715,11 @@ def page_content(menu: dict[str, Node], path: str, translation: Translation | No doc = E.article(class_="multicol") if rendered.multicol else E.article with doc: doc(HTML(rendered.html)) - _cards(doc, menu, node, path, translation) + _cards(doc, menu, data, node, path, translation) return HTML(str(doc)) -def _cards(doc, menu: dict[str, Node], node: Node, path: str, translation: Translation | None = None) -> None: +def _cards(doc, menu: dict[str, Node], data: Data, node: Node, path: str, translation: Translation | None = None) -> None: """Card stacks of the node's published children (nothing when childless). One column per direct child, all in a single full-width row (the .wide @@ -745,21 +745,21 @@ def _cards(doc, menu: dict[str, Node], node: Node, path: str, translation: Trans continue with doc.div(class_="stack"): for epath, enode in entries: - _card(doc, enode, epath, translation) + _card(doc, data, enode, epath, translation) def _walk(node: Node, path: str): """Published content pages of a subtree, pre-order in menu order: the node itself first when it has content (the stack's landing card), then its descendants (content-less nodes contribute only their subtree).""" - if node.content: + if node.chunks: yield path, node for slug, child in sorted_nodes(node.children): if child.published: yield from _walk(child, f"{path}/{slug}") -def _card(doc, node: Node, path: str, translation: Translation | None = None) -> None: +def _card(doc, data: Data, node: Node, path: str, translation: Translation | None = None) -> None: """One card in a stack: cover + title, plus the description when the page has no image (its card shows a gradient cover instead). @@ -767,8 +767,8 @@ def _card(doc, node: Node, path: str, translation: Translation | None = None) -> translated (only this page's own Markdown is translated); the title uses the translation's title map.""" image = description = "" - if node.content: - html = render(node.content, path, node.created, node.modified).html + if node.chunks: + html = render(node_markdown(data, node) or "", path, node.created, node.modified).html image, _ = _media(html) if not image: description = _description(html, 150) @@ -903,6 +903,7 @@ def _social_meta( def render_page( menu: dict[str, Node], + data: Data, path: str, brand: str = SITE_NAME, custom_css: str = "", @@ -923,13 +924,13 @@ def render_page( if translation is None: lang = i18n.ORIGINAL_LANGUAGE title = _title(path.rpartition("/")[2], node, translation, path) - main = page_content(menu, path, translation) + main = page_content(menu, data, path, translation) social = _social_meta(node, path, title, str(main), brand, base_url, lang) # hreflang alternates: every other language version (with ?lang=) plus # x-default for the plain URL. Emitted only when translations exist. alternates = [] if base_url: - available = i18n.available_languages(path) + available = i18n.available_languages(path, data) alternates = [ (l, f"{base_url}/{path}?lang={l}") for l in available if l != lang ] @@ -952,6 +953,7 @@ def render_page( def render_category( menu: dict[str, Node], + data: Data, path: str, brand: str = SITE_NAME, custom_css: str = "", @@ -973,7 +975,7 @@ def render_category( with doc: doc.h1(title) if any(c.published for c in node.children.values()): - _cards(doc, menu, node, path) + _cards(doc, menu, data, node, path) else: doc.p("This section has no page of its own yet.") return str( -- 2.55.0 From f63bb2c48276d3b75bc063333796600ef5c1a0fd Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:48:47 +0000 Subject: [PATCH 07/31] Mark phase-2 localization storage implemented; list chunks.py in AGENTS.md --- AGENTS.md | 2 ++ docs/localization.md | 6 +++--- docs/migrate.md | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 47df64c..30bc268 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,8 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `pagerite/` — Python backend package (hatchling build target). - `app.py` — FastAPI app and route registration. - `data.py` — msgspec Structs for the kanta database. + - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). + - `i18n.py` — language selection, translation assembly (chunks + patches). - `migrations.py` — kanta migrations (`migrate_vN`); ALL schema/storage upgrades live here (raw state dict before struct decoding), never in the app lifespan: v1 moves legacy in-db file blobs to the on-disk store and rebuilds the legacy flat `pages` as the menu tree, v2 rewrites `/_f/{hash}.ext` image links to the extension-less form, backfills AVIF/WebP/JPEG derivatives on disk and drops the obsolete `version` field. - `markdown.py` — markdown-it-py renderer. - `views.py` — shared page layout and rendering; theme/user-font resolution across `THEME_DIRS` / `FONT_DIRS` (cwd, site, platform data roots, then built-in `pagerite/themes/`, see `docs/themes-and-assets.md`). diff --git a/docs/localization.md b/docs/localization.md index 704336c..6da55c2 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -6,7 +6,7 @@ parameter or the `Accept-Language` header. - **Phase 1 (implemented):** negotiation, URL scheme, caching, rendering plumbing. Translations are consumed through a stub interface; the database still holds only the original language. -- **Phase 2 (final plan):** gettext-style fragment storage in the +- **Phase 2 (implemented):** gettext-style fragment storage in the database — machine-translated chunks plus user override patches, assembled at render time. Storage details in `docs/migrate.md`. @@ -66,7 +66,7 @@ Region tags normalize to their base subtag (`fi-FI` → `fi`). - The markdown typographer (SmartyPants) is English-centric; per-language typographer options are a possible follow-up, not blocking. -## Phase 2: fragment-based translation storage (draft) +## Phase 2: fragment-based translation storage (implemented) Phase 1 assumed whole-page translated Markdown delivered from outside. The refined model is gettext-style: an article has **one primary version** (its @@ -170,7 +170,7 @@ Full storage design and the `migrate_v3` restructuring live in - Article paths are stored and keyed **without leading slashes** (`"docs/setup"`, front page `""`); slashes are added only in hrefs. -### Render pipeline (replaces the phase-1 `get_translation` stub) +### Render pipeline (the phase-1 `get_translation` stub, now real) ```python def get_translation(path, lang, data) -> Translation | None: diff --git a/docs/migrate.md b/docs/migrate.md index 46d9493..1bf0380 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -1,6 +1,6 @@ # migrate_v3: content-addressed chunk storage -Status: **final plan**. `migrate_v3` restructures how article text and +Status: **implemented**. `migrate_v3` restructures how article text and translations are stored, motivated by the localization model in `docs/localization.md` (phase 2). Since it is a full migration, it is free to break the current `Node.content: str | None` layout. @@ -132,6 +132,21 @@ Chunking must be deterministic and shared with render/save, so `pagerite/chunks.py`) and are imported by both `migrations.py` and `views.py`/`app.py`. +## Implementation notes (deviations from the plan above) + +- Chunking lives in `pagerite/chunks.py`; hashing uses the `blake3` package + (already a dependency) with a 16-byte digest (`hexdigest(16)`). +- `Translation.titles` stayed keyed by node path (phase-1 shape, views + untouched): `get_translation` builds it by walking the menu with the same + per-title `trans[f"{chunk_key(node.title)}:{lang}"]` lookups. +- Insert hunks anchor on the whole preceding block (not just its tail) — + a stronger, simpler search context. +- `make_patch` diffs with `SequenceMatcher(autojunk=False)` so patches are + deterministic (popular lines like blank separators never become junk). +- Step 3's path normalization is a no-op in practice: the only path-keyed + store (`patches`) starts empty at v3; analytics paths live outside the + kantadb. The code still strips leading slashes defensively. + ## Garbage collection (later, manual or idle-time) Orphaned entries accumulate: chunks no longer referenced by any -- 2.55.0 From cc750236bc01c6d873fddc9dcea02c4b5ba0c252 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 02:09:27 +0000 Subject: [PATCH 08/31] Chunk keys as 9-byte bytes digests; trans nested by hash -> lang --- docs/localization.md | 17 +++++++++-------- docs/migrate.md | 30 ++++++++++++++++++------------ pagerite/chunks.py | 15 ++++++++++----- pagerite/data.py | 23 +++++++++++++---------- pagerite/i18n.py | 4 ++-- pagerite/migrations.py | 4 +++- 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/docs/localization.md b/docs/localization.md index 6da55c2..7f2cf1f 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -100,7 +100,7 @@ list blocks, tables, HTML blocks. A chunk's identity is its **source text**, gettext-msgid style: ```python -chunk_key = blake3(normalize(chunk_text), digest_size=16).hex() +chunk_key = blake3(normalize(chunk_text)).digest(9) # bytes; base64 at the JSON level ``` (`normalize`: strip trailing whitespace per line, collapse surrounding blank @@ -153,10 +153,11 @@ Full storage design and the `migrate_v3` restructuring live in `docs/migrate.md`. The short version, as it concerns this document: - Originals **and** translations are content-addressed text chunks in flat - stores: `Data.chunks: dict[hash, str]` and - `Data.trans: dict[f"{chunk_hash}:{lang}", str]` — path-independent, so - repeated paragraphs and menu titles are translated once and article moves - touch nothing. `Node.chunks: list[hash]` gives each article its order. + stores: `Data.chunks: dict[bytes, str]` and + `Data.trans: dict[bytes, dict[str, str]]` (chunk hash → lang → text) — + path-independent, so repeated paragraphs and menu titles are translated + once and article moves touch nothing. `Node.chunks: list[bytes]` gives + each article its order. - `Node` gains **`language: str = ""`**, inherited down the tree like `banner` (empty = nearest ancestor, front page last, site default `en` final). `select_language` and `` use the resolved value instead @@ -177,7 +178,7 @@ def get_translation(path, lang, data) -> Translation | None: if lang not in node.langs: return None hybrid = "\n\n".join( - chunks[h] if h in node.no_trans else trans.get(f"{h}:{lang}", chunks[h]) + chunks[h] if h in node.no_trans else trans.get(h, {}).get(lang, chunks[h]) for h in node.chunks ) for patch in data.patches.get(f"{path}:{lang}", []): @@ -191,8 +192,8 @@ def get_translation(path, lang, data) -> Translation | None: hreflang never probe the `trans` store chunk by chunk. A stale key is benign (the "translation" just renders as the original). - `titles` for nav/sidebar/cards: each node's translated title is - `trans.get(f"{hash(node.title)}:{lang}")` with per-node fallback — one dict - lookup per nav item at render time. + `trans.get(hash(node.title), {}).get(lang)` with per-node fallback — one + dict lookup per nav item at render time. - Cache invalidation: writes to `chunks` / `trans` / `patches` (translator, editor saves) call `_invalidate_pages()`, same as content writes. diff --git a/docs/migrate.md b/docs/migrate.md index 1bf0380..d5889c1 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -36,13 +36,13 @@ class Node(msgspec.Struct, omit_defaults=True): ... #: Replaces `content: str | None`. None = pure category label; #: a list (possibly empty) = a page, as ordered chunk hashes. - chunks: list[str] | None = None + chunks: list[bytes] | None = None #: Primary language of the article (BCP-47 base tag). "" = inherit #: (nearest ancestor, front page last, site default "en" final). language: str = "" #: Chunk hashes the editor marked "do not translate" (always served #: from the original). Presence-keys, value always True. - no_trans: dict[str, True] = {} + no_trans: dict[bytes, True] = {} #: Languages this article is available in (besides its primary #: language). Presence-keys, value always True — rendering, language #: selection and hreflang alternates read this set instead of probing @@ -52,12 +52,14 @@ class Node(msgspec.Struct, omit_defaults=True): class Data(msgspec.Struct): ... - #: All original-language text, content-addressed: blake3(normalized, - #: digest 16) hex -> Markdown chunk. Shared by every article. - chunks: dict[str, str] = {} - #: Machine translations: f"{chunk_hash}:{lang}" -> translated Markdown. + #: All original-language text, content-addressed: blake3(normalized) + #: digest[:9] -> Markdown chunk. Shared by every article. Keys are + #: bytes; kanta/msgspec base64-encode them at the JSON level. + chunks: dict[bytes, str] = {} + #: Machine translations: chunk hash -> lang -> translated Markdown + #: (nested, not tuple keys: msgspec's JSON serializer rejects them). #: Also used for node titles (hash of the title text). - trans: dict[str, str] = {} + trans: dict[bytes, dict[str, str]] = {} #: User override patches per article and language: #: f"{path}:{lang}" -> ordered patches (see localization.md). patches: dict[str, list[Patch]] = {} @@ -70,7 +72,7 @@ Notes: building hrefs. `migrate_v3` audits existing stored paths (translation keys, analytics references, any path-valued fields) and normalizes them. - **Titles are chunks too**, by hash only: the nav renderer looks up - `trans.get(f"{hash(node.title)}:{lang}")`. No separate title storage; + `trans.get(hash(node.title), {}).get(lang)`. No separate title storage; editing a title invalidates its translations automatically. - **Per-hunk options** live in two places: *inherent* options are derived at chunking time (code fences and HTML blocks are marked no-translate without @@ -88,7 +90,7 @@ that article rendering, `select_language`'s availability check, and hreflang alternate links never enumerate chunks. It is written by whoever writes translation data, in the same transaction: -- **Translator job:** after writing `trans[f"{h}:{lang}"]` entries for an +- **Translator job:** after writing `trans[h][lang]` entries for an article's chunks (or its title), set `node.langs[lang] = True`. - **Translated-view save:** appending the first patch for `f"{path}:{lang}"` sets `node.langs[lang] = True` (patches alone make the version exist). @@ -101,7 +103,7 @@ translation data, in the same transaction: - **Render:** `text = "\n\n".join(chunks[h] for h in node.chunks)` for the original; for language `L` (only ever attempted when `L in node.langs`), - per chunk `trans[f"{h}:{L}"]` unless missing or `h in node.no_trans`, + per chunk `trans.get(h, {}).get(L)` unless missing or `h in node.no_trans`, falling back to `chunks[h]`; then apply `patches.get(f"{path}:{L}", [])` in order (per-hunk, best effort); then `markdown.render` as today. All of this assembles the `Translation` the phase-1 plumbing already consumes. @@ -135,10 +137,14 @@ Chunking must be deterministic and shared with render/save, so ## Implementation notes (deviations from the plan above) - Chunking lives in `pagerite/chunks.py`; hashing uses the `blake3` package - (already a dependency) with a 16-byte digest (`hexdigest(16)`). + (already a dependency), truncated to a 9-byte `bytes` digest (kanta's + JSON persistence base64-encodes bytes keys to 12-char strings). +- `trans` is keyed `hash -> lang -> text` (nested dict), not by + `f"{hash}:{lang}"` tuples: msgspec's JSON serializer only supports + str-like/number-like dict keys, and kanta persists as JSON lines. - `Translation.titles` stayed keyed by node path (phase-1 shape, views untouched): `get_translation` builds it by walking the menu with the same - per-title `trans[f"{chunk_key(node.title)}:{lang}"]` lookups. + per-title `trans.get(chunk_key(node.title), {}).get(lang)` lookups. - Insert hunks anchor on the whole preceding block (not just its tail) — a stronger, simpler search context. - `make_patch` diffs with `SequenceMatcher(autojunk=False)` so patches are diff --git a/pagerite/chunks.py b/pagerite/chunks.py index 9d118e6..604bf8a 100644 --- a/pagerite/chunks.py +++ b/pagerite/chunks.py @@ -111,10 +111,15 @@ def _normalize(text: str) -> str: return "\n".join(line.rstrip() for line in text.split("\n")).strip("\n") -def chunk_key(text: str) -> str: - """Content key of a chunk: blake3 hex (16-byte digest, 32 hex chars) - of the normalized text — the same hasher app.py's file store uses.""" - return blake3.blake3(_normalize(text).encode()).hexdigest(16) +def chunk_key(text: str) -> bytes: + """Content key of a chunk: the first 9 bytes of the blake3 digest of + the normalized text (72 bits — a site's chunk count stays far below + the birthday bound), using the same hasher as app.py's file store. + + Keys are bytes: kanta/msgspec base64-encode them at the JSON + persistence level, so the raw database dicts carry 12-char strings. + """ + return blake3.blake3(_normalize(text).encode()).digest(9) def needs_translation(chunk: str) -> bool: @@ -137,7 +142,7 @@ def join_chunks(chunks: list[str]) -> str: return "\n\n".join(chunks) + "\n" if chunks else "" -def store_chunks(store: dict[str, str], markdown: str) -> list[str]: +def store_chunks(store: dict[bytes, str], markdown: str) -> list[bytes]: """Chunk ``markdown`` into ``store`` (hash -> text); return the ordered hashes. Unchanged chunks keep their hashes, so only genuinely new text lands in the kanta change diff. First writer wins: variants sharing a diff --git a/pagerite/data.py b/pagerite/data.py index 056b4ad..01c72bb 100644 --- a/pagerite/data.py +++ b/pagerite/data.py @@ -39,16 +39,16 @@ class Node(msgspec.Struct, omit_defaults=True): title: str = "" order: float = 0 - #: Ordered chunk hashes into ``Data.chunks``; None = pure category - #: label (its URL renders a placeholder page), a list (possibly - #: empty) = a page. - chunks: list[str] | None = None + #: Ordered chunk hashes (9-byte keys into ``Data.chunks``); None = + #: pure category label (its URL renders a placeholder page), a list + #: (possibly empty) = a page. + chunks: list[bytes] | None = None #: Primary language of the article (BCP-47 base tag). "" = inherit #: (nearest ancestor, front page last, site default "en" final). language: str = "" #: Chunk hashes the editor marked "do not translate" (always served #: from the original). Presence-keys, value always True. - no_trans: dict[str, bool] = {} + no_trans: dict[bytes, bool] = {} #: Languages this article is available in (besides its primary #: language). Presence-keys, value always True — the availability #: index for rendering, language selection and hreflang alternates; @@ -102,11 +102,14 @@ class Data(msgspec.Struct): #: /favicon.ico. favicon: str = "" #: All original-language page text, content-addressed: - #: chunk_key -> Markdown chunk. Shared by every article. - chunks: dict[str, str] = {} - #: Machine translations: f"{chunk_hash}:{lang}" -> translated - #: Markdown. Also used for node titles (hash of the title text). - trans: dict[str, str] = {} + #: chunk_key (9 bytes; base64 at the JSON level) -> Markdown chunk. + #: Shared by every article. + chunks: dict[bytes, str] = {} + #: Machine translations: chunk hash -> lang -> translated Markdown + #: (a nested dict rather than tuple keys, which msgspec's JSON + #: serializer does not support). Also used for node titles (hash of + #: the title text). + trans: dict[bytes, dict[str, str]] = {} #: User override patches per article and language: #: f"{path}:{lang}" -> ordered patches (paths without leading slash). patches: dict[str, list[Patch]] = {} diff --git a/pagerite/i18n.py b/pagerite/i18n.py index 6ae7802..d239bb0 100644 --- a/pagerite/i18n.py +++ b/pagerite/i18n.py @@ -133,7 +133,7 @@ def hybrid_markdown(data: Data, node: Node, path: str, lang: str) -> str: hybrid = join_chunks([ data.chunks.get(h, "") if h in node.no_trans - else data.trans.get(f"{h}:{lang}") or data.chunks.get(h, "") + else data.trans.get(h, {}).get(lang) or data.chunks.get(h, "") for h in node.chunks or [] ]) for patch in data.patches.get(f"{path}:{lang}", []): @@ -154,7 +154,7 @@ def title_map(data: Data, lang: str) -> dict[str, str]: for slug, node in nodes.items(): path = f"{prefix}/{slug}" if prefix else slug if node.title: - t = data.trans.get(f"{chunk_key(node.title)}:{lang}") + t = data.trans.get(chunk_key(node.title), {}).get(lang) if t: titles[path] = t walk(node.children, path) diff --git a/pagerite/migrations.py b/pagerite/migrations.py index ed61ad2..e3ba86c 100644 --- a/pagerite/migrations.py +++ b/pagerite/migrations.py @@ -141,6 +141,8 @@ def migrate_v3(d: dict) -> None: list as ``chunks`` (an absent content stays absent, i.e. None = a pure category label; "" chunks to an empty list = an empty page). + Chunk keys are 9-byte blake3 digests; at this raw JSON level they are + base64 strings (decoding into the structs restores ``bytes`` keys). ``trans``/``patches`` start empty; the translator job fills them and maintains the ``langs`` index as translations land. ``language``, ``no_trans`` and ``langs`` need nothing — struct defaults cover them. @@ -155,7 +157,7 @@ def migrate_v3(d: dict) -> None: if isinstance(content, str): hashes = [] for chunk in chunk_markdown(content): - key = chunk_key(chunk) + key = base64.b64encode(chunk_key(chunk)).decode() store.setdefault(key, chunk) hashes.append(key) node["chunks"] = hashes -- 2.55.0 From 10c4c9c4f966f232246feba4f653cbb140500e5c Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 02:16:28 +0000 Subject: [PATCH 09/31] Translator service API: GET/POST /_api/translate/{lang} --- docs/localization.md | 7 ++- docs/migrate.md | 7 ++- pagerite/app.py | 121 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 4 deletions(-) diff --git a/docs/localization.md b/docs/localization.md index 7f2cf1f..1964cd2 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -213,8 +213,11 @@ def get_translation(path, lang, data) -> Translation | None: ### Explicitly out of scope for phase 2 - The machine translation itself: chunking output goes in, translated chunks - come back. A background job writes `trans` entries; this doc only defines - the storage key (`chunk_hash:lang`) and the merge semantics. + come back. The **service API exists** — `GET /_api/translate/{lang}` lists + pending items (`{"key", "text", "path", "kind"}`, key = base64 chunk hash), + `POST /_api/translate/{lang}` stores a batch (`{"items": [{key, text}]}`) + into `trans` and maintains `node.langs`; an external service does the + actual translating (gated by the /_api forward-auth like everything else). - Garbage collection of orphaned chunks/translations (see docs/migrate.md). - sitemap.xml per-language entries; translated UI chrome; per-language typographer options; multi-locale date/number formatting. diff --git a/docs/migrate.md b/docs/migrate.md index d5889c1..865551b 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -91,7 +91,12 @@ alternate links never enumerate chunks. It is written by whoever writes translation data, in the same transaction: - **Translator job:** after writing `trans[h][lang]` entries for an - article's chunks (or its title), set `node.langs[lang] = True`. + article's chunks (or its title), set `node.langs[lang] = True`. The + translation service API does both: `GET /_api/translate/{lang}` lists + pending items (titles + translatable chunks lacking an entry, deduped by + hash), `POST /_api/translate/{lang}` stores a batch into `trans`, sets + `langs` on every article that gained an entry and invalidates the page + cache — all in one transaction. - **Translated-view save:** appending the first patch for `f"{path}:{lang}"` sets `node.langs[lang] = True` (patches alone make the version exist). - **Removals:** deleting a patch or GC'ing translations re-derives the key: diff --git a/pagerite/app.py b/pagerite/app.py index 31155f0..852db30 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -13,6 +13,7 @@ walking the tree (``resolve``), moves are slot detach/attach """ import asyncio +import base64 import gzip import ipaddress import mimetypes @@ -49,7 +50,7 @@ from zstandard import ZstdCompressor from pagerite import analytics, i18n, seed, views from pagerite.__main__ import DEVMODE -from pagerite.chunks import store_chunks +from pagerite.chunks import chunk_key, needs_translation, store_chunks from pagerite.data import ( Data, Node, @@ -965,6 +966,124 @@ async def delete_page(path: str) -> None: _invalidate_pages() +def _check_translate_lang(lang: str) -> str: + """Normalize a ``/_api/translate`` language parameter; translations never + target the original language (its text lives in the chunk store, and an + "en" langs key would advertise a bogus ?lang=en alternate).""" + tag = i18n.base_tag(lang) + if not tag or tag == i18n.ORIGINAL_LANGUAGE: + raise HTTPException(400, "bad target language") + return tag + + +@app.get("/_api/translate/{lang}") +async def translate_pending(lang: str) -> list[dict]: + """Pending translation items for ``lang`` (translation service API). + + Every page node (published or not) contributes its title and each of + its chunks that needs translation (``needs_translation``), is not + editor-flagged no-translate (``node.no_trans``) and has no ``trans`` + entry for ``lang`` yet. Each item is + ``{"key", "text", "path", "kind"}``: ``key`` is the base64 of the + 9-byte chunk hash — the handle the service translates against and + POSTs back; ``text`` the original Markdown (or title); ``path`` the + article it came from (no leading slash); ``kind`` "chunk" or "title". + Items are deduped by key: content-addressed text (shared paragraphs, + repeated titles) is translated once, whichever page it first came from. + """ + lang = _check_translate_lang(lang) + items = [] + seen = set() + + def emit(key: bytes, text: str, path: str, kind: str) -> None: + if key in seen or lang in data.trans.get(key, {}): + return + seen.add(key) + items.append({ + "key": base64.b64encode(key).decode(), + "text": text, + "path": path, + "kind": kind, + }) + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.chunks is not None: + if node.title: + emit(chunk_key(node.title), node.title, path, "title") + for h in node.chunks: + text = data.chunks.get(h) + if ( + text is not None + and h not in node.no_trans + and needs_translation(text) + ): + emit(h, text, path, "chunk") + walk(node.children, path) + + walk(data.menu, "") + return items + + +class TranslationItemIn(BaseModel): + """One translated fragment submitted by the translation service.""" + + key: str # base64 of the 9-byte chunk hash, as issued by the GET + text: str # the translated Markdown (or title) + + +class TranslationsIn(BaseModel): + """Batch of translations for one language.""" + + items: list[TranslationItemIn] + + +@app.post("/_api/translate/{lang}") +async def translate_submit(lang: str, body: TranslationsIn) -> dict: + """Store a batch of machine translations for ``lang``, one transaction. + + Keys are the base64 chunk hashes issued by the GET; each text lands in + ``trans[key][lang]``. Unknown keys are stored anyway (unreferenced + hashes are never read, and the content may simply have moved on since + the GET); duplicates within a batch overwrite, last wins; a malformed + base64 key rejects the whole batch with 400. Every article that gained + at least one entry gets ``node.langs[lang]`` set (the availability + index), and cached pages are invalidated. Returns the stored count and + the paths of the articles that gained the language. + """ + lang = _check_translate_lang(lang) + try: + entries = [ + (base64.b64decode(item.key, validate=True), item.text) + for item in body.items + ] + except ValueError: + raise HTTPException(400, "malformed base64 key") from None + with kanta.transaction("submit translations", extra=lang): + stored = set() + for key, text in entries: + data.trans.setdefault(key, {})[lang] = text + stored.add(key) + pages = [] + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.chunks is not None: + keys = set(node.chunks) + if node.title: + keys.add(chunk_key(node.title)) + if keys & stored: + node.langs[lang] = True + pages.append(path) + walk(node.children, path) + + walk(data.menu, "") + _invalidate_pages() + return {"stored": len(entries), "pages": pages} + + _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") -- 2.55.0 From 69962df0c2611c13be3252b27c904969beb4ed98 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 03:03:44 +0000 Subject: [PATCH 10/31] Translator WS protocol structs + pending/store core, Data.translate_key --- pagerite/data.py | 4 ++ pagerite/translate.py | 126 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 pagerite/translate.py diff --git a/pagerite/data.py b/pagerite/data.py index 01c72bb..f7f0f3e 100644 --- a/pagerite/data.py +++ b/pagerite/data.py @@ -101,6 +101,10 @@ class Data(msgspec.Struct): #: linked as on every page. Empty = the build's #: /favicon.ico. favicon: str = "" + #: API key gating the translator service WebSocket (/_translate/{key}; + #: the external forward-auth does not cover that route). Generated + #: lazily at startup when empty (see lifespan in app.py). + translate_key: str = "" #: All original-language page text, content-addressed: #: chunk_key (9 bytes; base64 at the JSON level) -> Markdown chunk. #: Shared by every article. diff --git a/pagerite/translate.py b/pagerite/translate.py new file mode 100644 index 0000000..7328dd1 --- /dev/null +++ b/pagerite/translate.py @@ -0,0 +1,126 @@ +"""Translator service protocol and its transport-independent core. + +The external machine-translation service connects over WebSocket +(``/_translate/``, see app.py) and exchanges JSON frames decoded into +the tagged msgspec structs below (``bytes`` fields ride as base64 — no +manual encoding anywhere). This module holds the message structs plus the +two computations shared by the WS handler: which fragments are pending for +a language (``pending_items``) and storing a batch of results +(``store_results``). +""" + +import msgspec + +from pagerite.chunks import chunk_key, needs_translation +from pagerite.data import Data, Node, sorted_nodes + + +class Hello(msgspec.Struct, tag="hello"): + """Client greeting on connect: the target languages it handles.""" + + langs: list[str] + + +class TransItem(msgspec.Struct): + """One fragment to translate: original Markdown (or a node title).""" + + key: bytes #: 9-byte chunk hash (base64 in the JSON frame) + text: str + path: str #: article it came from ("" = front page), no leading slash + kind: str #: "chunk" | "title" + + +class Job(msgspec.Struct, tag="job"): + """Server push: pending items for one language.""" + + lang: str + items: list[TransItem] + + +class TransResult(msgspec.Struct): + """One translated fragment.""" + + key: bytes + text: str + + +class Result(msgspec.Struct, tag="result"): + """Client reply: a batch of translations for one language.""" + + lang: str + items: list[TransResult] + + +#: Union of the client -> server frames (the "type" tag selects). +ClientMsg = Hello | Result + + +def pending_items(data: Data, lang: str) -> list[TransItem]: + """Fragments of the site still untranslated for ``lang``, deduped by key. + + Every page node (published or not) contributes its title and each chunk + that needs translation (``needs_translation``), is not editor-flagged + no-translate (``node.no_trans``) and has no ``trans`` entry for ``lang`` + yet. Content-addressed text (shared paragraphs, repeated titles) appears + once, under the first page in menu order that has it. + """ + items: list[TransItem] = [] + seen: set[bytes] = set() + + def emit(key: bytes, text: str, path: str, kind: str) -> None: + if key in seen or lang in data.trans.get(key, {}): + return + seen.add(key) + items.append(TransItem(key=key, text=text, path=path, kind=kind)) + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.chunks is not None: + if node.title: + emit(chunk_key(node.title), node.title, path, "title") + for h in node.chunks: + text = data.chunks.get(h) + if ( + text is not None + and h not in node.no_trans + and needs_translation(text) + ): + emit(h, text, path, "chunk") + walk(node.children, path) + + walk(data.menu, "") + return items + + +def store_results(data: Data, lang: str, items: list[TransResult]) -> list[str]: + """Store a batch of machine translations for ``lang``; return the paths + of the articles that gained at least one entry. + + Pure data operations: the caller wraps this in a kanta transaction and + invalidates pages. Unknown keys are stored anyway (unreferenced hashes + are never read, and the content may simply have moved on since the job + was pushed); duplicates within a batch overwrite, last wins. Every + article that gained an entry gets ``node.langs[lang]`` set (the + availability index, docs/migrate.md) — because chunks are + content-addressed, that includes pages merely sharing a fragment. + """ + stored = {item.key for item in items} + for item in items: + data.trans.setdefault(item.key, {})[lang] = item.text + pages: list[str] = [] + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.chunks is not None: + keys = set(node.chunks) + if node.title: + keys.add(chunk_key(node.title)) + if keys & stored: + node.langs[lang] = True + pages.append(path) + walk(node.children, path) + + walk(data.menu, "") + return pages -- 2.55.0 From 979504e52676af591b1745e3d646f8547d74f33d Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 03:22:00 +0000 Subject: [PATCH 11/31] Translator API as authed WebSocket /_translate/{key} with delta job push --- pagerite/app.py | 239 +++++++++++++++++++++++++----------------------- 1 file changed, 125 insertions(+), 114 deletions(-) diff --git a/pagerite/app.py b/pagerite/app.py index 852db30..4b71801 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -13,12 +13,12 @@ walking the tree (``resolve``), moves are slot detach/attach """ import asyncio -import base64 import gzip import ipaddress import mimetypes import os import re +import secrets import shutil import socket import tempfile @@ -48,9 +48,9 @@ from mediapreview import dispatch from pydantic import BaseModel from zstandard import ZstdCompressor -from pagerite import analytics, i18n, seed, views +from pagerite import analytics, i18n, seed, translate, views from pagerite.__main__ import DEVMODE -from pagerite.chunks import chunk_key, needs_translation, store_chunks +from pagerite.chunks import store_chunks from pagerite.data import ( Data, Node, @@ -287,6 +287,10 @@ def _seed(data: Data) -> None: async def lifespan(_app: FastAPI) -> AsyncIterator[None]: """Open the database (migrations run inside kanta.open), load assets, load GeoIP.""" await kanta.open() + # The translator service key is persisted; generated once on first boot. + if not data.translate_key: + with kanta.transaction("generate translate key"): + data.translate_key = secrets.token_urlsafe(24) await asyncio.to_thread(file_store.load) await frontend.load() # Decompress/open the DB-IP MMDB once at startup. Lookups are then @@ -405,10 +409,12 @@ _render_gen = 0 def _invalidate_pages() -> None: - """Drop cached page bodies and bump the render generation (ETags).""" + """Drop cached page bodies and bump the render generation (ETags); + any content change also pushes translation-job deltas to translators.""" global _render_gen _render_gen += 1 _cached_body.cache_clear() + _schedule_translator_notify() @lru_cache(maxsize=128) @@ -616,7 +622,7 @@ async def update_structure(op: StructureOp) -> None: async def get_settings() -> dict: """Site-wide settings (brand, theme, custom CSS and favicon URL), plus the themes, banner designs and user fonts available on disk for the - selectors.""" + selectors and the translator service key (for the /_translate socket).""" return { "brand": data.brand, "brand_html": data.brand_html, @@ -628,6 +634,7 @@ async def get_settings() -> dict: "fonts": views._user_fonts(), "transition": data.transition, "transitions": views._transition_names(), + "translate_key": data.translate_key, } @@ -966,122 +973,126 @@ async def delete_page(path: str) -> None: _invalidate_pages() -def _check_translate_lang(lang: str) -> str: - """Normalize a ``/_api/translate`` language parameter; translations never - target the original language (its text lives in the chunk store, and an - "en" langs key would advertise a bogus ?lang=en alternate).""" - tag = i18n.base_tag(lang) - if not tag or tag == i18n.ORIGINAL_LANGUAGE: - raise HTTPException(400, "bad target language") - return tag +class _TranslatorState: + """One connected translator socket: announced languages and the + (lang, chunk-key) pairs already sent and still outstanding on it. + + Per-connection only: a reconnecting client re-receives everything + pending for its languages (its Hello triggers a full Job push).""" + + def __init__(self, langs: set[str]) -> None: + self.langs = langs + self.outstanding: set[tuple[str, bytes]] = set() -@app.get("/_api/translate/{lang}") -async def translate_pending(lang: str) -> list[dict]: - """Pending translation items for ``lang`` (translation service API). +#: Connected translator sockets and their per-connection state. +_translator_clients: dict[WebSocket, _TranslatorState] = {} - Every page node (published or not) contributes its title and each of - its chunks that needs translation (``needs_translation``), is not - editor-flagged no-translate (``node.no_trans``) and has no ``trans`` - entry for ``lang`` yet. Each item is - ``{"key", "text", "path", "kind"}``: ``key`` is the base64 of the - 9-byte chunk hash — the handle the service translates against and - POSTs back; ``text`` the original Markdown (or title); ``path`` the - article it came from (no leading slash); ``kind`` "chunk" or "title". - Items are deduped by key: content-addressed text (shared paragraphs, - repeated titles) is translated once, whichever page it first came from. + +def _schedule_translator_notify() -> None: + """Schedule a delta Job push to connected translators, if any. + + The hook is _invalidate_pages (sync, called inside transactions): the + task first runs once the current coroutine awaits again, i.e. after the + transaction has committed. No-op without a running loop (CLI use). """ - lang = _check_translate_lang(lang) - items = [] - seen = set() - - def emit(key: bytes, text: str, path: str, kind: str) -> None: - if key in seen or lang in data.trans.get(key, {}): - return - seen.add(key) - items.append({ - "key": base64.b64encode(key).decode(), - "text": text, - "path": path, - "kind": kind, - }) - - def walk(nodes: dict[str, Node], prefix: str) -> None: - for slug, node in sorted_nodes(nodes): - path = f"{prefix}/{slug}" if prefix else slug - if node.chunks is not None: - if node.title: - emit(chunk_key(node.title), node.title, path, "title") - for h in node.chunks: - text = data.chunks.get(h) - if ( - text is not None - and h not in node.no_trans - and needs_translation(text) - ): - emit(h, text, path, "chunk") - walk(node.children, path) - - walk(data.menu, "") - return items - - -class TranslationItemIn(BaseModel): - """One translated fragment submitted by the translation service.""" - - key: str # base64 of the 9-byte chunk hash, as issued by the GET - text: str # the translated Markdown (or title) - - -class TranslationsIn(BaseModel): - """Batch of translations for one language.""" - - items: list[TranslationItemIn] - - -@app.post("/_api/translate/{lang}") -async def translate_submit(lang: str, body: TranslationsIn) -> dict: - """Store a batch of machine translations for ``lang``, one transaction. - - Keys are the base64 chunk hashes issued by the GET; each text lands in - ``trans[key][lang]``. Unknown keys are stored anyway (unreferenced - hashes are never read, and the content may simply have moved on since - the GET); duplicates within a batch overwrite, last wins; a malformed - base64 key rejects the whole batch with 400. Every article that gained - at least one entry gets ``node.langs[lang]`` set (the availability - index), and cached pages are invalidated. Returns the stored count and - the paths of the articles that gained the language. - """ - lang = _check_translate_lang(lang) + if not _translator_clients: + return try: - entries = [ - (base64.b64decode(item.key, validate=True), item.text) - for item in body.items - ] - except ValueError: - raise HTTPException(400, "malformed base64 key") from None - with kanta.transaction("submit translations", extra=lang): - stored = set() - for key, text in entries: - data.trans.setdefault(key, {})[lang] = text - stored.add(key) - pages = [] + asyncio.get_running_loop() + except RuntimeError: + return + asyncio.create_task(_notify_translators()) - def walk(nodes: dict[str, Node], prefix: str) -> None: - for slug, node in sorted_nodes(nodes): - path = f"{prefix}/{slug}" if prefix else slug - if node.chunks is not None: - keys = set(node.chunks) - if node.title: - keys.add(chunk_key(node.title)) - if keys & stored: - node.langs[lang] = True - pages.append(path) - walk(node.children, path) - walk(data.menu, "") - _invalidate_pages() - return {"stored": len(entries), "pages": pages} +async def _notify_translators() -> None: + """Push newly-pending items to every connected translator, deltas only: + items never sent on that connection and still untranslated.""" + for ws, state in list(_translator_clients.items()): + for lang in sorted(state.langs): + items = [ + item + for item in translate.pending_items(data, lang) + if (lang, item.key) not in state.outstanding + ] + if not items: + continue + state.outstanding |= {(lang, item.key) for item in items} + try: + job = translate.Job(lang=lang, items=items) + await ws.send_text(msgspec.json.encode(job).decode()) + except Exception: # send failed: the receive loop cleans up + _translator_clients.pop(ws, None) + break + + +@app.websocket("/_translate/{clientkey}") +async def translate_ws(ws: WebSocket, clientkey: str) -> None: + """Translator service channel (docs/localization.md). + + Deliberately NOT under /_api/: the external forward-auth is skipped; + the server-generated client key in the path is the access control + (``Data.translate_key``, generated at startup, shown in the admin's + /_api/settings). A wrong/empty key rejects the handshake — closing + before accept makes Starlette answer HTTP 403. + + Protocol (JSON frames, msgspec structs in translate.py): the client + opens with Hello(langs) (normalized to base subtags; "en"/empty + dropped); the server pushes Job(lang, items) — on connect everything + pending per language, afterwards deltas on content change — and the + client answers with Result(lang, items) batches. A Result for an + unannounced or invalid language, or any malformed frame, closes the + socket with a protocol error. + """ + if not data.translate_key or clientkey != data.translate_key: + await ws.close(code=1008) # policy violation; pre-accept = HTTP 403 + return + await ws.accept() + state: _TranslatorState | None = None + try: + while True: + raw = await ws.receive_text() + try: + msg = msgspec.json.decode(raw.encode(), type=translate.ClientMsg) + except msgspec.DecodeError: + await ws.close(code=1002) # protocol error + return + if isinstance(msg, translate.Hello): + if state is not None: # one Hello per connection + await ws.close(code=1002) + return + langs = { + tag + for lang in msg.langs + if (tag := i18n.base_tag(lang)) and tag != i18n.ORIGINAL_LANGUAGE + } + state = _TranslatorState(langs) + _translator_clients[ws] = state + for lang in sorted(langs): + items = translate.pending_items(data, lang) + if not items: + continue + state.outstanding |= {(lang, item.key) for item in items} + job = translate.Job(lang=lang, items=items) + await ws.send_text(msgspec.json.encode(job).decode()) + else: # translate.Result + lang = i18n.base_tag(msg.lang) + if ( + state is None # results before Hello + or not lang + or lang == i18n.ORIGINAL_LANGUAGE + or lang not in state.langs # not announced in Hello + ): + await ws.close(code=1002) + return + with kanta.transaction("translator results", extra=lang): + translate.store_results(data, lang, msg.items) + _invalidate_pages() + state.outstanding -= {(lang, item.key) for item in msg.items} + except WebSocketDisconnect: + pass + finally: + _translator_clients.pop(ws, None) _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") -- 2.55.0 From 2f78877f151e74c7ca5d13cc3f701b790635459e Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 03:24:57 +0000 Subject: [PATCH 12/31] Seed-X translator service client for the /_translate socket --- scripts/translator.py | 179 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 scripts/translator.py diff --git a/scripts/translator.py b/scripts/translator.py new file mode 100644 index 0000000..f640ef4 --- /dev/null +++ b/scripts/translator.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.14" +# dependencies = [ +# "accelerate>=1.14.0", +# "msgspec>=0.19.0", +# "torch>=2.13.0", +# "tracerite>=2.6.5", +# "transformers>=5.16.1", +# "websockets>=15.0.1", +# ] +# /// +"""Pagerite translator service: translate site content with Seed-X-PPO-7B. + +Connects to a Pagerite server's translator WebSocket (``/_translate/`` — +deliberately outside /_api, the key is the access control; find it in the +site settings, GET /_api/settings -> ``translate_key``), announces the +languages it handles and translates whatever the server pushes: everything +pending on connect, then deltas as the content changes. Results go back per +job and the server stores them (docs/localization.md). + +Seed-X-PPO-7B (bf16, ~15 GB) is the only supported model; its 28 languages +are the ceiling of what a site can announce through this script. + +Usage: + uv run scripts/translator.py ws://localhost:8080 --key "..." --to finnish + uv run scripts/translator.py wss://example.com --key "..." --to fi german es +""" + +import argparse +import asyncio +import sys +import time + +import msgspec +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer +import tracerite +import websockets + +tracerite.load() + +SEED_X = "ByteDance-Seed/Seed-X-PPO-7B" + +# Seed-X language tags (appended to the prompt; required by its PPO training) +SEED_X_TAGS = { + "arabic": "ar", "chinese": "zh", "czech": "cs", "danish": "da", + "dutch": "nl", "english": "en", "finnish": "fi", "french": "fr", + "german": "de", "greek": "el", "hungarian": "hu", "indonesian": "id", + "italian": "it", "japanese": "ja", "korean": "ko", "malay": "ms", + "norwegian": "no", "persian": "fa", "polish": "pl", "portuguese": "pt", + "romanian": "ro", "russian": "ru", "spanish": "es", "swedish": "sv", + "thai": "th", "turkish": "tr", "ukrainian": "uk", "vietnamese": "vi", +} +SEED_X_NAMES = {v: k for k, v in SEED_X_TAGS.items()} + +#: The fragments are Markdown; Seed-X has no system prompt, so it goes in-line. +NOTE = ", preserving all Markdown formatting, URLs and code exactly unchanged" + + +# The message structs below duplicate pagerite/translate.py 1:1: this script +# runs in its own uv environment and cannot import the server package. The +# "type" tag selects the frame; bytes fields ride as base64. +class Hello(msgspec.Struct, tag="hello"): + """Client greeting on connect: the target languages it handles.""" + + langs: list[str] + + +class TransItem(msgspec.Struct): + """One fragment to translate: original Markdown (or a node title).""" + + key: bytes #: 9-byte chunk hash (base64 in the JSON frame) + text: str + path: str #: article it came from ("" = front page), no leading slash + kind: str #: "chunk" | "title" + + +class Job(msgspec.Struct, tag="job"): + """Server push: pending items for one language.""" + + lang: str + items: list[TransItem] + + +class TransResult(msgspec.Struct): + """One translated fragment.""" + + key: bytes + text: str + + +class Result(msgspec.Struct, tag="result"): + """Client reply: a batch of translations for one language.""" + + lang: str + items: list[TransResult] + + +def load_seed_x(): + t0 = time.monotonic() + tokenizer = AutoTokenizer.from_pretrained(SEED_X) + model = AutoModelForCausalLM.from_pretrained(SEED_X, dtype=torch.bfloat16, device_map="auto") + print(f"[seed-x loaded in {time.monotonic() - t0:.0f}s]", file=sys.stderr) + return tokenizer, model + + +def seed_x_chunk(tokenizer, model, text: str, target_lang: str, tag: str, source_lang: str = "English", + note: str = ""): + """Translate one segment; returns (translation, output_tokens, generation_seconds).""" + # No chat template on this model; the trailing language tag is required (trans/ style prompt). + prompt = f"Translate the following {source_lang} text into {target_lang}{note}:\n{text} <{tag}>" + inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + t0 = time.monotonic() + out = model.generate(**inputs, max_new_tokens=max(1024, 2 * inputs.input_ids.shape[1]), do_sample=False) + dt = time.monotonic() - t0 + n = out.shape[1] - inputs.input_ids.shape[1] + return tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True).strip(), n, dt + + +async def do_job(ws, job: Job, tokenizer, model) -> None: + """Translate every item of one job and send the results back as a batch.""" + lang_name = SEED_X_NAMES[job.lang].capitalize() + results = [] + for i, item in enumerate(job.items, 1): + # Deliberately blocking: nothing else needs the loop while a job is + # being answered, and the reconnect loop recovers a dropped connection + # (results already stored stay stored; only the still-pending items + # are re-pushed). + text, tokens, dt = seed_x_chunk(tokenizer, model, item.text, lang_name, job.lang, note=NOTE) + results.append(TransResult(key=item.key, text=text)) + print(f"[{job.lang} {i}/{len(job.items)} {item.kind} {item.path or '/'}: " + f"{tokens} tokens in {dt:.1f}s = {tokens / dt:.1f} tok/s]", file=sys.stderr) + await ws.send(msgspec.json.encode(Result(lang=job.lang, items=results)).decode()) + + +async def serve(url: str, codes: list[str], tokenizer, model) -> None: + """Connect, announce languages, answer jobs; reconnect with backoff.""" + backoff = 1 + while True: + try: + async with websockets.connect(url) as ws: + backoff = 1 + await ws.send(msgspec.json.encode(Hello(langs=codes)).decode()) + print(f"[connected; translating: {', '.join(codes)}]", file=sys.stderr) + async for raw in ws: + await do_job(ws, msgspec.json.decode(raw, type=Job), tokenizer, model) + except websockets.exceptions.InvalidHandshake: + sys.exit("handshake rejected; check --key and the server URL") + except (OSError, websockets.exceptions.ConnectionClosed) as e: + print(f"[connection lost ({e}); reconnecting in {backoff}s]", file=sys.stderr) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("server", help="server WebSocket base URL, e.g. ws://localhost:8080") + p.add_argument("--key", required=True, help="translator API key (site settings: translate_key)") + p.add_argument("--to", required=True, nargs="+", + help="target language(s): Seed-X names or codes, e.g. finnish de es") + args = p.parse_args() + + codes = [] + for lang in args.to: + low = lang.lower() + code = SEED_X_TAGS.get(low) or (low if low in SEED_X_NAMES else None) + if code is None: + p.error(f"unknown language {lang!r}; supported: {', '.join(sorted(SEED_X_TAGS))}") + if code not in codes: + codes.append(code) + + tokenizer, model = load_seed_x() # once, before the (re)connect loop + url = f"{args.server.rstrip('/')}/_translate/{args.key}" + asyncio.run(serve(url, codes, tokenizer, model)) + + +if __name__ == "__main__": + main() -- 2.55.0 From f6eef1c75fb190bfaa4c1ff5da5ca63d259e03e4 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 03:30:36 +0000 Subject: [PATCH 13/31] Document the translator WebSocket API and reference client --- AGENTS.md | 4 +++- docs/localization.md | 36 ++++++++++++++++++++++++++++++------ docs/migrate.md | 16 +++++++++------- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 30bc268..7a15ffb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,7 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `data.py` — msgspec Structs for the kanta database. - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). - `i18n.py` — language selection, translation assembly (chunks + patches). + - `translate.py` — translator service protocol (msgspec structs) and pending/store core for the `/_translate/{key}` WebSocket (docs/localization.md). - `migrations.py` — kanta migrations (`migrate_vN`); ALL schema/storage upgrades live here (raw state dict before struct decoding), never in the app lifespan: v1 moves legacy in-db file blobs to the on-disk store and rebuilds the legacy flat `pages` as the menu tree, v2 rewrites `/_f/{hash}.ext` image links to the extension-less form, backfills AVIF/WebP/JPEG derivatives on disk and drops the obsolete `version` field. - `markdown.py` — markdown-it-py renderer. - `views.py` — shared page layout and rendering; theme/user-font resolution across `THEME_DIRS` / `FONT_DIRS` (cwd, site, platform data roots, then built-in `pagerite/themes/`, see `docs/themes-and-assets.md`). @@ -25,6 +26,7 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `pagerite.js` — public page entry. - `assets/` — base CSS, Pygments styles, fonts. - `scripts/devserver.py` — dev server with auto reload (the user mostly uses this; avoid running the server yourself, ask the user to test). +- `scripts/translator.py` — Seed-X translator service client for the `/_translate/{key}` socket (reference client, runs in its own uv env via PEP 723). Server run by CLI entry point `uv run pagerite` (no auto reloads, build needed). Dev mode is `scripts/devserver.py` (auto reloads, no build needed). @@ -55,5 +57,5 @@ Server run by CLI entry point `uv run pagerite` (no auto reloads, build needed). - Keep dependencies minimal; add via `uv add` and mention it. - The public URL space belongs to content (pretty slugs at root). Reserve only `/_` for the machinery (`/_api/`, `/_f/`, `/_assets/`), plus `/favicon.ico` from the build. Slugs are lowercase ASCII letters, digits, hyphens and underscores `[a-z0-9_-]` (the site editor filters input live via `slugify.js`, built on the `transliteration` npm package — unicode folds to ASCII, spaces become hyphens; an empty slug on a new page is derived from its title), may not begin with `_` or `.`, and such URLs are never looked up as content. -- No auth in core code; the SSO/reverse proxy gates all of `/_api` (forward-auth) and owns `/auth/` (login/logout, session validation). Pages render identically for everyone; pagerite.js adds the editing UI only after the auth server validates the session. +- No auth in core code; the SSO/reverse proxy gates all of `/_api` (forward-auth) and owns `/auth/` (login/logout, session validation). Pages render identically for everyone; pagerite.js adds the editing UI only after the auth server validates the session. The one keyed exception is `/_translate/{key}` (translator service; `Data.translate_key`, see docs/localization.md). - Update the relevant MarkDown files when architecture, tooling, or conventions change. diff --git a/docs/localization.md b/docs/localization.md index 1964cd2..b00a16a 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -210,14 +210,38 @@ def get_translation(path, lang, data) -> Translation | None: updates `Data.chunks` / `node.chunks` — only genuinely new text lands in the kanta change diff (see docs/migrate.md). +### Translator service API + +An external machine-translation service connects over WebSocket at +`/_translate/{key}` — deliberately **not** under `/_api`: the SSO +forward-auth does not cover that route, and the key in the path is the +access control. The key is `Data.translate_key`, generated once at startup +and surfaced to the admin in `GET /_api/settings` as `translate_key`. A +wrong or empty key rejects the handshake (close-before-accept → HTTP 403). + +Frames are JSON-encoded tagged msgspec structs (`pagerite/translate.py`; +`bytes` fields ride as base64): + +- `{"type": "hello", "langs": [...]}` — client greeting: the target + languages it handles (normalized to base subtags; `en`/empty dropped). +- `{"type": "job", "lang", "items": [{key, text, path, kind}]}` — server + push: pending fragments (article titles and chunks), deduped by key. +- `{"type": "result", "lang", "items": [{key, text}]}` — client reply: + translated fragments, matched to content by chunk key alone. + +The model is **push**, not polling: on `hello` the server sends everything +pending per announced language; afterwards `_invalidate_pages()` (called by +every content/translation write) schedules a delta push of newly pending +items. Outstanding-item tracking is per connection, so a reconnecting client +simply re-receives everything still pending. Results are stored into `trans` +in one transaction and set `node.langs[lang]` on every article they touch +(shared chunks make several pages gain a language from one fragment). + ### Explicitly out of scope for phase 2 -- The machine translation itself: chunking output goes in, translated chunks - come back. The **service API exists** — `GET /_api/translate/{lang}` lists - pending items (`{"key", "text", "path", "kind"}`, key = base64 chunk hash), - `POST /_api/translate/{lang}` stores a batch (`{"items": [{key, text}]}`) - into `trans` and maintains `node.langs`; an external service does the - actual translating (gated by the /_api forward-auth like everything else). +- The machine translation itself: the API above moves fragments in and out; + the translating is external. `scripts/translator.py` is the reference + client (Seed-X-PPO-7B only — its 28 languages are the ceiling). - Garbage collection of orphaned chunks/translations (see docs/migrate.md). - sitemap.xml per-language entries; translated UI chrome; per-language typographer options; multi-locale date/number formatting. diff --git a/docs/migrate.md b/docs/migrate.md index 865551b..f55bc54 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -52,6 +52,9 @@ class Node(msgspec.Struct, omit_defaults=True): class Data(msgspec.Struct): ... + #: API key gating the translator service WebSocket (/_translate/{key}); + #: generated lazily at startup (see the lifespan in app.py). + translate_key: str = "" #: All original-language text, content-addressed: blake3(normalized) #: digest[:9] -> Markdown chunk. Shared by every article. Keys are #: bytes; kanta/msgspec base64-encode them at the JSON level. @@ -90,13 +93,12 @@ that article rendering, `select_language`'s availability check, and hreflang alternate links never enumerate chunks. It is written by whoever writes translation data, in the same transaction: -- **Translator job:** after writing `trans[h][lang]` entries for an - article's chunks (or its title), set `node.langs[lang] = True`. The - translation service API does both: `GET /_api/translate/{lang}` lists - pending items (titles + translatable chunks lacking an entry, deduped by - hash), `POST /_api/translate/{lang}` stores a batch into `trans`, sets - `langs` on every article that gained an entry and invalidates the page - cache — all in one transaction. +- **Translator service:** the WebSocket API at `/_translate/{key}` (see + docs/localization.md) pushes pending fragments (titles + translatable + chunks lacking an entry, deduped by hash) and receives result batches; + storing a batch writes `trans[h][lang]` entries, sets + `node.langs[lang] = True` on every article that gained one and + invalidates the page cache — all in one transaction. - **Translated-view save:** appending the first patch for `f"{path}:{lang}"` sets `node.langs[lang] = True` (patches alone make the version exist). - **Removals:** deleting a patch or GC'ing translations re-derives the key: -- 2.55.0 From 1ebd789220a43aed2e4731c0d342b46193b1ffdf Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 04:08:19 +0000 Subject: [PATCH 14/31] Translation dispatcher: single-item jobs, capability Hello, wanted-langs setting --- pagerite/app.py | 150 ++++++++++++++++++++++++++---------------- pagerite/data.py | 5 ++ pagerite/translate.py | 34 ++++++---- 3 files changed, 122 insertions(+), 67 deletions(-) diff --git a/pagerite/app.py b/pagerite/app.py index 4b71801..2845791 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -410,11 +410,11 @@ _render_gen = 0 def _invalidate_pages() -> None: """Drop cached page bodies and bump the render generation (ETags); - any content change also pushes translation-job deltas to translators.""" + any content change also re-runs translation dispatch.""" global _render_gen _render_gen += 1 _cached_body.cache_clear() - _schedule_translator_notify() + _schedule_translation_dispatch() @lru_cache(maxsize=128) @@ -622,7 +622,8 @@ async def update_structure(op: StructureOp) -> None: async def get_settings() -> dict: """Site-wide settings (brand, theme, custom CSS and favicon URL), plus the themes, banner designs and user fonts available on disk for the - selectors and the translator service key (for the /_translate socket).""" + selectors, the translator service key and the wanted translation + languages (for the /_translate socket).""" return { "brand": data.brand, "brand_html": data.brand_html, @@ -635,6 +636,7 @@ async def get_settings() -> dict: "transition": data.transition, "transitions": views._transition_names(), "translate_key": data.translate_key, + "translate_langs": sorted(data.translate_langs), } @@ -646,6 +648,7 @@ class SettingsIn(BaseModel): custom_css: str brand_html: str = "" transition: str = "cube" + translate_langs: list[str] | None = None # None keeps the current set @app.put("/_api/settings", status_code=204) @@ -657,6 +660,12 @@ async def put_settings(settings: SettingsIn) -> None: data.theme = settings.theme data.custom_css = settings.custom_css data.transition = settings.transition + if settings.translate_langs is not None: + data.translate_langs = { + tag: True + for lang in settings.translate_langs + if (tag := i18n.base_tag(lang)) and tag != i18n.ORIGINAL_LANGUAGE + } _invalidate_pages() @@ -973,28 +982,37 @@ async def delete_page(path: str) -> None: _invalidate_pages() +# WebSocket API for external translation services (not under /_api: it is keyed +# with Data.translate_key instead of the SSO forward-auth). The server is a +# dispatcher: one single-item job at a time per connection, offered in the +# intersection of the wanted languages (Data.translate_langs) and the +# connection's announced capabilities. Results are matched to content by +# chunk key alone. class _TranslatorState: - """One connected translator socket: announced languages and the - (lang, chunk-key) pairs already sent and still outstanding on it. + """One connected translator socket: the language codes it announced as + capabilities (Hello) and the (lang, chunk-key) job currently in flight + on it — one at a time, the next is sent only after its Result. - Per-connection only: a reconnecting client re-receives everything - pending for its languages (its Hello triggers a full Job push).""" + Per-connection only: in-flight lives solely here, so on disconnect the + item simply becomes pending again and is re-offered to any free capable + connection.""" - def __init__(self, langs: set[str]) -> None: - self.langs = langs - self.outstanding: set[tuple[str, bytes]] = set() + def __init__(self, capable: set[str]) -> None: + self.capable = capable + self.inflight: tuple[str, bytes] | None = None #: Connected translator sockets and their per-connection state. _translator_clients: dict[WebSocket, _TranslatorState] = {} -def _schedule_translator_notify() -> None: - """Schedule a delta Job push to connected translators, if any. +def _schedule_translation_dispatch() -> None: + """Schedule a dispatch pass, if any translator is connected. - The hook is _invalidate_pages (sync, called inside transactions): the - task first runs once the current coroutine awaits again, i.e. after the - transaction has committed. No-op without a running loop (CLI use). + The content-change hook is _invalidate_pages (sync, called inside + transactions): the task first runs once the current coroutine awaits + again, i.e. after the transaction has committed. No-op without a + running loop (CLI use). """ if not _translator_clients: return @@ -1002,28 +1020,49 @@ def _schedule_translator_notify() -> None: asyncio.get_running_loop() except RuntimeError: return - asyncio.create_task(_notify_translators()) + asyncio.create_task(_dispatch_translations()) -async def _notify_translators() -> None: - """Push newly-pending items to every connected translator, deltas only: - items never sent on that connection and still untranslated.""" +async def _dispatch_translations() -> None: + """Offer one pending item to every free capable connection. + + Runs on every relevant event: Hello, Result, disconnect and content + change (via _invalidate_pages). A connection with no wanted ∩ capable + overlap simply stays idle. Pending is derived from the trans store + (translate.pending_items) minus the items in flight on any connection. + """ + wanted = { + tag + for lang in data.translate_langs + if (tag := i18n.base_tag(lang)) and tag != i18n.ORIGINAL_LANGUAGE + } + if not wanted: + return for ws, state in list(_translator_clients.items()): - for lang in sorted(state.langs): - items = [ - item - for item in translate.pending_items(data, lang) - if (lang, item.key) not in state.outstanding - ] - if not items: - continue - state.outstanding |= {(lang, item.key) for item in items} - try: - job = translate.Job(lang=lang, items=items) - await ws.send_text(msgspec.json.encode(job).decode()) - except Exception: # send failed: the receive loop cleans up - _translator_clients.pop(ws, None) + if state.inflight is not None: + continue + langs = wanted & state.capable + if not langs: + continue + inflight = {s.inflight for s in _translator_clients.values() if s.inflight} + job = None + for lang in sorted(langs): + for item in translate.pending_items(data, lang): + if (lang, item.key) not in inflight: + job = translate.Job( + lang=lang, key=item.key, text=item.text, + path=item.path, kind=item.kind, + ) + break + if job is not None: break + if job is None: + continue + state.inflight = (job.lang, job.key) # before the await: no double-assign + try: + await ws.send_text(msgspec.json.encode(job).decode()) + except Exception: # send failed: the receive loop cleans up + _translator_clients.pop(ws, None) @app.websocket("/_translate/{clientkey}") @@ -1037,12 +1076,13 @@ async def translate_ws(ws: WebSocket, clientkey: str) -> None: before accept makes Starlette answer HTTP 403. Protocol (JSON frames, msgspec structs in translate.py): the client - opens with Hello(langs) (normalized to base subtags; "en"/empty - dropped); the server pushes Job(lang, items) — on connect everything - pending per language, afterwards deltas on content change — and the - client answers with Result(lang, items) batches. A Result for an - unannounced or invalid language, or any malformed frame, closes the - socket with a protocol error. + opens with Hello(langs) announcing its CAPABILITIES — the language + codes its model can produce (normalized to base subtags; "en"/empty + dropped). The dispatcher sends one Job(lang, key, text, path, kind) + at a time and waits for the matching Result(lang, key, text) before + offering the next. A Result without an in-flight job or with a + different (lang, key), a duplicate Hello, or any malformed frame + closes the socket with a protocol error. """ if not data.translate_key or clientkey != data.translate_key: await ws.close(code=1008) # policy violation; pre-accept = HTTP 403 @@ -1061,38 +1101,36 @@ async def translate_ws(ws: WebSocket, clientkey: str) -> None: if state is not None: # one Hello per connection await ws.close(code=1002) return - langs = { + state = _TranslatorState({ tag for lang in msg.langs if (tag := i18n.base_tag(lang)) and tag != i18n.ORIGINAL_LANGUAGE - } - state = _TranslatorState(langs) + }) _translator_clients[ws] = state - for lang in sorted(langs): - items = translate.pending_items(data, lang) - if not items: - continue - state.outstanding |= {(lang, item.key) for item in items} - job = translate.Job(lang=lang, items=items) - await ws.send_text(msgspec.json.encode(job).decode()) + _schedule_translation_dispatch() else: # translate.Result lang = i18n.base_tag(msg.lang) if ( state is None # results before Hello - or not lang - or lang == i18n.ORIGINAL_LANGUAGE - or lang not in state.langs # not announced in Hello + or state.inflight is None # no job in flight + or (lang, msg.key) != state.inflight # wrong job ): await ws.close(code=1002) return with kanta.transaction("translator results", extra=lang): - translate.store_results(data, lang, msg.items) - _invalidate_pages() - state.outstanding -= {(lang, item.key) for item in msg.items} + paths = translate.store_results( + data, lang, [translate.TransResult(key=msg.key, text=msg.text)] + ) + _invalidate_pages() # schedules the next dispatch + state.inflight = None + if paths: + print(f"[{lang}] now available for {len(paths)} page(s): {', '.join(sorted(paths))}") except WebSocketDisconnect: pass finally: - _translator_clients.pop(ws, None) + if _translator_clients.pop(ws, None) is not None: + # The in-flight item (if any) is pending again; offer it around. + _schedule_translation_dispatch() _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") diff --git a/pagerite/data.py b/pagerite/data.py index f7f0f3e..d7978b0 100644 --- a/pagerite/data.py +++ b/pagerite/data.py @@ -105,6 +105,11 @@ class Data(msgspec.Struct): #: the external forward-auth does not cover that route). Generated #: lazily at startup when empty (see lifespan in app.py). translate_key: str = "" + #: Wanted target languages for the translator service (presence-keys, + #: value always True). The dispatcher offers jobs only in the + #: intersection of these and a connection's announced capabilities. + #: Read/set via /_api/settings (no editing UI yet). + translate_langs: dict[str, bool] = {} #: All original-language page text, content-addressed: #: chunk_key (9 bytes; base64 at the JSON level) -> Markdown chunk. #: Shared by every article. diff --git a/pagerite/translate.py b/pagerite/translate.py index 7328dd1..08ba109 100644 --- a/pagerite/translate.py +++ b/pagerite/translate.py @@ -5,8 +5,9 @@ The external machine-translation service connects over WebSocket the tagged msgspec structs below (``bytes`` fields ride as base64 — no manual encoding anywhere). This module holds the message structs plus the two computations shared by the WS handler: which fragments are pending for -a language (``pending_items``) and storing a batch of results -(``store_results``). +a language (``pending_items``) and storing a result (``store_results``). +The dispatcher itself (one job at a time per connection, wanted ∩ capable +language matching, requeue on disconnect) lives in app.py. """ import msgspec @@ -16,7 +17,9 @@ from pagerite.data import Data, Node, sorted_nodes class Hello(msgspec.Struct, tag="hello"): - """Client greeting on connect: the target languages it handles.""" + """Client greeting on connect: the language codes its model CAN produce + (capabilities). The server offers jobs only in the intersection with + the wanted target languages (``Data.translate_langs``).""" langs: list[str] @@ -31,24 +34,33 @@ class TransItem(msgspec.Struct): class Job(msgspec.Struct, tag="job"): - """Server push: pending items for one language.""" + """Server push: ONE fragment to translate. + + Exactly one job is in flight per connection — the next is sent only + after this one's Result. Clients wanting parallelism open multiple + connections.""" lang: str - items: list[TransItem] + key: bytes #: 9-byte chunk hash (base64 in the JSON frame) + text: str + path: str #: article it came from ("" = front page), no leading slash + kind: str #: "chunk" | "title" class TransResult(msgspec.Struct): - """One translated fragment.""" + """One translated fragment (storage level, see store_results).""" key: bytes text: str class Result(msgspec.Struct, tag="result"): - """Client reply: a batch of translations for one language.""" + """Client reply: the translation of the connection's current Job + (must match its lang and key exactly).""" lang: str - items: list[TransResult] + key: bytes + text: str #: Union of the client -> server frames (the "type" tag selects). @@ -94,13 +106,13 @@ def pending_items(data: Data, lang: str) -> list[TransItem]: def store_results(data: Data, lang: str, items: list[TransResult]) -> list[str]: - """Store a batch of machine translations for ``lang``; return the paths - of the articles that gained at least one entry. + """Store machine translations for ``lang``; return the paths of the + articles that gained at least one entry. Pure data operations: the caller wraps this in a kanta transaction and invalidates pages. Unknown keys are stored anyway (unreferenced hashes are never read, and the content may simply have moved on since the job - was pushed); duplicates within a batch overwrite, last wins. Every + was pushed); re-storing an existing key overwrites, last wins. Every article that gained an entry gets ``node.langs[lang]`` set (the availability index, docs/migrate.md) — because chunks are content-addressed, that includes pages merely sharing a fragment. -- 2.55.0 From 0610daf1afc63463d8fb932bd4d96b115d3de238 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 04:10:06 +0000 Subject: [PATCH 15/31] Translator client: full-URL arg, capability Hello, one-job-at-a-time --- scripts/translator.py | 104 +++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 62 deletions(-) diff --git a/scripts/translator.py b/scripts/translator.py index f640ef4..0f12bb9 100644 --- a/scripts/translator.py +++ b/scripts/translator.py @@ -12,19 +12,20 @@ # /// """Pagerite translator service: translate site content with Seed-X-PPO-7B. -Connects to a Pagerite server's translator WebSocket (``/_translate/`` — -deliberately outside /_api, the key is the access control; find it in the -site settings, GET /_api/settings -> ``translate_key``), announces the -languages it handles and translates whatever the server pushes: everything -pending on connect, then deltas as the content changes. Results go back per -job and the server stores them (docs/localization.md). +Connects to a Pagerite server's translator WebSocket — the full URL +including the access key (the admin finds it in the site settings, +GET /_api/settings -> ``translate_key``) — and announces the languages the +model CAN translate (capabilities). The server dispatches one single-item +job at a time per connection, offered only in its configured target +languages (``Data.translate_langs``) ∩ the announced capabilities; a +dropped connection's in-flight item is simply re-offered +(docs/localization.md). For parallelism, run multiple instances. -Seed-X-PPO-7B (bf16, ~15 GB) is the only supported model; its 28 languages -are the ceiling of what a site can announce through this script. +Seed-X-PPO-7B (bf16, ~15 GB) is the only supported model. Usage: - uv run scripts/translator.py ws://localhost:8080 --key "..." --to finnish - uv run scripts/translator.py wss://example.com --key "..." --to fi german es + uv run scripts/translator.py ws://localhost:8410/_translate/KEY + uv run scripts/translator.py wss://example.com/_translate/KEY """ import argparse @@ -58,45 +59,37 @@ SEED_X_NAMES = {v: k for k, v in SEED_X_TAGS.items()} NOTE = ", preserving all Markdown formatting, URLs and code exactly unchanged" -# The message structs below duplicate pagerite/translate.py 1:1: this script -# runs in its own uv environment and cannot import the server package. The +# The wire structs below duplicate pagerite/translate.py: this script runs +# in its own uv environment and cannot import the server package. The # "type" tag selects the frame; bytes fields ride as base64. class Hello(msgspec.Struct, tag="hello"): - """Client greeting on connect: the target languages it handles.""" + """Client greeting on connect: the language codes its model CAN produce + (capabilities). The server offers jobs only in the intersection with + its wanted target languages.""" langs: list[str] -class TransItem(msgspec.Struct): - """One fragment to translate: original Markdown (or a node title).""" +class Job(msgspec.Struct, tag="job"): + """Server push: ONE fragment to translate. Exactly one job is in flight + per connection — the next arrives only after this one's Result.""" + lang: str key: bytes #: 9-byte chunk hash (base64 in the JSON frame) text: str path: str #: article it came from ("" = front page), no leading slash kind: str #: "chunk" | "title" -class Job(msgspec.Struct, tag="job"): - """Server push: pending items for one language.""" +class Result(msgspec.Struct, tag="result"): + """Client reply: the translation of the connection's current Job + (must match its lang and key exactly).""" lang: str - items: list[TransItem] - - -class TransResult(msgspec.Struct): - """One translated fragment.""" - key: bytes text: str -class Result(msgspec.Struct, tag="result"): - """Client reply: a batch of translations for one language.""" - - lang: str - items: list[TransResult] - - def load_seed_x(): t0 = time.monotonic() tokenizer = AutoTokenizer.from_pretrained(SEED_X) @@ -119,34 +112,31 @@ def seed_x_chunk(tokenizer, model, text: str, target_lang: str, tag: str, source async def do_job(ws, job: Job, tokenizer, model) -> None: - """Translate every item of one job and send the results back as a batch.""" + """Translate the job's one fragment and send the result back.""" lang_name = SEED_X_NAMES[job.lang].capitalize() - results = [] - for i, item in enumerate(job.items, 1): - # Deliberately blocking: nothing else needs the loop while a job is - # being answered, and the reconnect loop recovers a dropped connection - # (results already stored stay stored; only the still-pending items - # are re-pushed). - text, tokens, dt = seed_x_chunk(tokenizer, model, item.text, lang_name, job.lang, note=NOTE) - results.append(TransResult(key=item.key, text=text)) - print(f"[{job.lang} {i}/{len(job.items)} {item.kind} {item.path or '/'}: " - f"{tokens} tokens in {dt:.1f}s = {tokens / dt:.1f} tok/s]", file=sys.stderr) - await ws.send(msgspec.json.encode(Result(lang=job.lang, items=results)).decode()) + # Deliberately blocking: nothing else needs the loop while the job is + # being answered, and the reconnect loop recovers a dropped connection + # (the in-flight item is simply re-offered). + text, tokens, dt = seed_x_chunk(tokenizer, model, job.text, lang_name, job.lang, note=NOTE) + print(f"[{job.lang} {job.kind} {job.path or '/'}: " + f"{tokens} tokens in {dt:.1f}s = {tokens / dt:.1f} tok/s]", file=sys.stderr) + await ws.send(msgspec.json.encode(Result(lang=job.lang, key=job.key, text=text)).decode()) -async def serve(url: str, codes: list[str], tokenizer, model) -> None: - """Connect, announce languages, answer jobs; reconnect with backoff.""" +async def serve(url: str, tokenizer, model) -> None: + """Connect, announce capabilities, answer jobs; reconnect with backoff.""" backoff = 1 while True: try: async with websockets.connect(url) as ws: backoff = 1 - await ws.send(msgspec.json.encode(Hello(langs=codes)).decode()) - print(f"[connected; translating: {', '.join(codes)}]", file=sys.stderr) + await ws.send(msgspec.json.encode(Hello(langs=sorted(SEED_X_NAMES))).decode()) + print(f"[connected; announced {len(SEED_X_NAMES)} language capabilities]", + file=sys.stderr) async for raw in ws: await do_job(ws, msgspec.json.decode(raw, type=Job), tokenizer, model) except websockets.exceptions.InvalidHandshake: - sys.exit("handshake rejected; check --key and the server URL") + sys.exit("handshake rejected; check the URL (including the key)") except (OSError, websockets.exceptions.ConnectionClosed) as e: print(f"[connection lost ({e}); reconnecting in {backoff}s]", file=sys.stderr) await asyncio.sleep(backoff) @@ -155,24 +145,14 @@ async def serve(url: str, codes: list[str], tokenizer, model) -> None: def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("server", help="server WebSocket base URL, e.g. ws://localhost:8080") - p.add_argument("--key", required=True, help="translator API key (site settings: translate_key)") - p.add_argument("--to", required=True, nargs="+", - help="target language(s): Seed-X names or codes, e.g. finnish de es") + p.add_argument("url", help="full translator WebSocket URL including the key, " + "e.g. ws://localhost:8410/_translate/KEY") args = p.parse_args() - - codes = [] - for lang in args.to: - low = lang.lower() - code = SEED_X_TAGS.get(low) or (low if low in SEED_X_NAMES else None) - if code is None: - p.error(f"unknown language {lang!r}; supported: {', '.join(sorted(SEED_X_TAGS))}") - if code not in codes: - codes.append(code) + if not args.url.startswith(("ws://", "wss://")): + p.error("url must start with ws:// or wss://") tokenizer, model = load_seed_x() # once, before the (re)connect loop - url = f"{args.server.rstrip('/')}/_translate/{args.key}" - asyncio.run(serve(url, codes, tokenizer, model)) + asyncio.run(serve(args.url, tokenizer, model)) if __name__ == "__main__": -- 2.55.0 From dd0a6cc04b60529f28ec6c2b606dee8d72ff8399 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 04:11:40 +0000 Subject: [PATCH 16/31] Document the translation dispatcher and capability handshake --- docs/localization.md | 45 +++++++++++++++++++++++++++++++------------- docs/migrate.md | 15 ++++++++++----- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/docs/localization.md b/docs/localization.md index b00a16a..07a9c0e 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -222,20 +222,39 @@ wrong or empty key rejects the handshake (close-before-accept → HTTP 403). Frames are JSON-encoded tagged msgspec structs (`pagerite/translate.py`; `bytes` fields ride as base64): -- `{"type": "hello", "langs": [...]}` — client greeting: the target - languages it handles (normalized to base subtags; `en`/empty dropped). -- `{"type": "job", "lang", "items": [{key, text, path, kind}]}` — server - push: pending fragments (article titles and chunks), deduped by key. -- `{"type": "result", "lang", "items": [{key, text}]}` — client reply: - translated fragments, matched to content by chunk key alone. +- `{"type": "hello", "langs": [...]}` — client greeting announcing its + **capabilities**: the language codes its model can produce (normalized + to base subtags; `en`/empty dropped). +- `{"type": "job", "lang", "key", "text", "path", "kind"}` — server push: + ONE fragment to translate (an article title or a chunk). +- `{"type": "result", "lang", "key", "text"}` — client reply: the + translation of the connection's current job, matching it by (lang, key). -The model is **push**, not polling: on `hello` the server sends everything -pending per announced language; afterwards `_invalidate_pages()` (called by -every content/translation write) schedules a delta push of newly pending -items. Outstanding-item tracking is per connection, so a reconnecting client -simply re-receives everything still pending. Results are stored into `trans` -in one transaction and set `node.langs[lang]` on every article they touch -(shared chunks make several pages gain a language from one fragment). +Which languages get translated is **server-configured**: +`Data.translate_langs` (presence-key dict, read/set via `/_api/settings` +as `translate_langs`; no editing UI yet). The dispatcher offers a +connection jobs only in `wanted ∩ capable`; a connection without overlap +simply stays idle. + +Dispatch semantics (all in app.py): + +- **One job at a time per connection** — the next job is sent only after + the current one's result. Clients wanting parallelism open multiple + connections (e.g. several `scripts/translator.py` instances). +- Pending work is derived from the `trans` store + (`translate.pending_items`) minus the items in flight on any connection, + so a **disconnect requeues** that connection's in-flight item and it is + offered to any free capable connection. +- Dispatch re-runs on every relevant event: Hello, result, disconnect and + content change (`_invalidate_pages()` schedules it, so the pass runs + after the writing transaction commits). +- A result with no job in flight, a mismatched (lang, key), a duplicate + hello, or any malformed frame closes the socket with a protocol error. + +Results are stored into `trans` in one transaction and set +`node.langs[lang]` on every article they touch (shared chunks make several +pages gain a language from one fragment). Unknown keys are stored anyway +and re-storing overwrites — results are idempotent. ### Explicitly out of scope for phase 2 diff --git a/docs/migrate.md b/docs/migrate.md index f55bc54..721c121 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -55,6 +55,9 @@ class Data(msgspec.Struct): #: API key gating the translator service WebSocket (/_translate/{key}); #: generated lazily at startup (see the lifespan in app.py). translate_key: str = "" + #: Wanted target languages for the translator service (presence-keys); + #: jobs are offered only in these ∩ a connection's capabilities. + translate_langs: dict[str, True] = {} #: All original-language text, content-addressed: blake3(normalized) #: digest[:9] -> Markdown chunk. Shared by every article. Keys are #: bytes; kanta/msgspec base64-encode them at the JSON level. @@ -94,11 +97,13 @@ alternate links never enumerate chunks. It is written by whoever writes translation data, in the same transaction: - **Translator service:** the WebSocket API at `/_translate/{key}` (see - docs/localization.md) pushes pending fragments (titles + translatable - chunks lacking an entry, deduped by hash) and receives result batches; - storing a batch writes `trans[h][lang]` entries, sets - `node.langs[lang] = True` on every article that gained one and - invalidates the page cache — all in one transaction. + docs/localization.md) offers pending fragments (titles + translatable + chunks lacking an entry for the language) as single-item jobs — one at + a time per connection, in `Data.translate_langs` ∩ the connection's + announced capabilities — and receives the matching result; storing it + writes the `trans[h][lang]` entry, sets `node.langs[lang] = True` on + every article that gained one and invalidates the page cache — all in + one transaction. - **Translated-view save:** appending the first patch for `f"{path}:{lang}"` sets `node.langs[lang] = True` (patches alone make the version exist). - **Removals:** deleting a patch or GC'ing translations re-derives the key: -- 2.55.0 From 14a37f5ab33a0b40a67d12c222bf7af78344a9f0 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 15:08:42 +0000 Subject: [PATCH 17/31] Sticky ?lang= navigation + site-wide hreflang/canonical tags - Server replicates a ?lang= override onto the navigation links it renders (nav, sidebar, cards, brand), so clicks and prefetches stay in the chosen language even without JS; link_lang is part of the ETag and body cache key (query and header renders of the same language differ in their links). - pagerite.js drops the Accept-Language header hack: the remembered language rides internal fetches as ?lang= instead (added when a link lacks one), the page cache keys on path+query, and history/address bar keep the pretty query-less URL. - Canonical names the actually served language (plain URL for the original, ?lang= for translations); hreflang alternates are site-wide from translate_langs, identical on every page: x-default (the plain autodetecting URL) first, then every language explicitly, default included, emitted right after canonical before the social tags. --- docs/localization.md | 38 ++++++++++----- docs/migrate.md | 5 +- frontend/src/pagerite.js | 73 ++++++++++++++++++--------- pagerite/app.py | 29 +++++++---- pagerite/data.py | 4 +- pagerite/i18n.py | 9 ---- pagerite/views.py | 103 +++++++++++++++++++++++---------------- 7 files changed, 160 insertions(+), 101 deletions(-) diff --git a/docs/localization.md b/docs/localization.md index 07a9c0e..0e20c3c 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -36,15 +36,24 @@ Region tags normalize to their base subtag (`fi-FI` → `fi`). - Canonical URLs stay pretty (`/some-page`). Each language version is addressable as `/some-page?lang=fi` so search engines can index them. -- `` points to the page **itself including the query** - (each language version is its own canonical). -- `` entries point to every other language - version (with `?lang=`), plus `x-default` for the plain URL. -- On page load, pagerite.js removes the `?lang=` query via - `history.replaceState` (restoring the pretty URL) but remembers the language - in a JS variable. All fetch-navigation and preloads it performs afterwards - send that language in the `Accept-Language` header, so the chosen language - sticks for the session of clicks. +- `` names the **actually served language**: the plain + URL when serving the original (for SEO the non-query URL means the + article's own language), `?lang=xx` when serving a translation — however + the language was arrived at (query or header). +- `` entries follow the canonical + directly (before the social meta tags) and are the same set on every + page — the site-wide configured languages (`translate_langs`, which the + translator works to fill in): `x-default` first, pointing at the plain + autodetecting URL, then every language explicitly with `?lang=`, the + default language included. +- The override sticks for the session of clicks: a page requested with + `?lang=` replicates the query onto the navigation links it renders (nav, + sidebar, cards, brand — in-article links are content and stay as + authored), so plain clicks and no-JS navigation keep the language. + pagerite.js additionally strips the query from the address bar via + `history.replaceState` (pretty, shareable URLs), remembers the language, + and adds it to every internal fetch that lacks one (preloads, + fetch-navigations, history traversals); history entries stay query-less. - A full page refresh or a shared link resets to automatic selection (header only). This gives a clean one-time override without cookies. @@ -53,7 +62,10 @@ Region tags normalize to their base subtag (`fi-FI` → `fi`). - Content responses carry `Vary: accept-language` (added to the existing `accept-encoding` vary). - `_cached_body` and the page ETag include the **selected language** (not the - raw header, which would blow up the cache key space). + raw header, which would blow up the cache key space) and the **replicated + link language**: a `?lang=fi` render and a header-selected Finnish render + of the same page differ in their navigation links, so they are cached as + separate variants. - `` reflects the served language. ### Rendering @@ -188,9 +200,9 @@ def get_translation(path, lang, data) -> Translation | None: - Availability is an article-level index: `node.langs: dict[lang, True]`, maintained by the translation writers (translator job, patch saves) in the - same transaction as their data writes — rendering, language selection and - hreflang never probe the `trans` store chunk by chunk. A stale key is - benign (the "translation" just renders as the original). + same transaction as their data writes — rendering and language selection + never probe the `trans` store chunk by chunk. A stale key is benign (the + "translation" just renders as the original). - `titles` for nav/sidebar/cards: each node's translated title is `trans.get(hash(node.title), {}).get(lang)` with per-node fallback — one dict lookup per nav item at render time. diff --git a/docs/migrate.md b/docs/migrate.md index 721c121..78ebef5 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -119,8 +119,9 @@ translation data, in the same transaction: falling back to `chunks[h]`; then apply `patches.get(f"{path}:{L}", [])` in order (per-hunk, best effort); then `markdown.render` as today. All of this assembles the `Translation` the phase-1 plumbing already consumes. -- **Availability:** `available_languages(path)` = `sorted(node.langs)`; - hreflang alternates and `?lang=` handling use exactly this set. +- **Availability:** `node.langs` is the availability index; `?lang=` + handling uses exactly this set. (hreflang alternates are site-wide from + `translate_langs` instead — see docs/localization.md.) - **Save (primary language):** server re-chunks the submitted Markdown, inserts new hashes into `Data.chunks`, replaces `node.chunks`. Unchanged chunks keep their hashes — only genuinely new text lands in the diff. diff --git a/frontend/src/pagerite.js b/frontend/src/pagerite.js index 5b95d6f..71e1cee 100644 --- a/frontend/src/pagerite.js +++ b/frontend/src/pagerite.js @@ -36,21 +36,43 @@ import "overlayscrollbars/overlayscrollbars.css"; // --- Language override (?lang=) --------------------------------------- // /page?lang=fi serves a translated, indexable version (each language is - // its own canonical). Restore the pretty URL on load but remember the - // language: all fetch-navigation and preload requests below send it as - // Accept-Language, so the chosen language sticks for the session of - // clicks. A full refresh or shared link resets to automatic selection - // (the browser's own Accept-Language). See docs/localization.md. + // its own canonical). The chosen language sticks for the session of + // clicks: the server replicates ?lang= onto the navigation links it + // renders (nav, sidebar, cards — in-article links are content and stay + // as authored), and pageUrl adds it to internal fetches that lack one. + // The address bar keeps the pretty URL: the query is stripped on load + // and never pushed into history. A full refresh or a shared link resets + // to automatic selection (the browser's own Accept-Language — every + // plain fetch carries it by default). See docs/localization.md. const langParam = new URL(location.href).searchParams.get("lang"); if (langParam) { const url = new URL(location.href); url.searchParams.delete("lang"); history.replaceState(history.state, "", url); } - const pageHeaders = langParam ? { "Accept-Language": langParam } : {}; - // The in-memory page cache is keyed per language: the same pathname holds - // different HTML for each selected language. - const cacheKey = (pathname) => (langParam ? `${langParam}|${pathname}` : pathname); + // An internal URL as fetched: carries the session's ?lang= unless the + // link already pins a language of its own. With no ?lang= on the initial + // load nothing is ever added. + const pageUrl = (url) => { + const u = new URL(url, location.href); + if (langParam && u.origin === location.origin && !u.searchParams.has("lang")) { + u.searchParams.set("lang", langParam); + } + return u; + }; + // The in-memory page cache is keyed by path + query: the same pathname + // holds different HTML for each language version. + const rawKey = (url) => { + const u = new URL(url, location.href); + return u.pathname + u.search; + }; + const cacheKey = (url) => rawKey(pageUrl(url)); + // What goes into the address bar and history: the pretty URL, no ?lang=. + const prettyUrl = (url) => { + const u = pageUrl(url); + u.searchParams.delete("lang"); + return u; + }; // Regions every page has. #sidebar is NOT among them: it is omitted // entirely when the section has no sub-navigation, and handled below. @@ -370,9 +392,13 @@ import "overlayscrollbars/overlayscrollbars.css"; // received it as the document (re-fetching would be redundant, and // browser heuristics may send it without if-none-match, defeating the // conditional request); it enters the cache when navigated to. - const pageCache = new Map(); // cacheKey(pathname) -> HTML text + const pageCache = new Map(); // rawKey/cacheKey(url) -> HTML text addEventListener("pagerite:page-fetched", (ev) => { - pageCache.set(cacheKey(new URL(ev.detail.url, location.href).pathname), ev.detail.html); + // The editors' re-renders fetch the plain URL (no ?lang=); key by the + // URL as announced. Adding the session language would cache that + // header-language copy under the translated page's key and serve it + // back on navigation. + pageCache.set(rawKey(ev.detail.url), ev.detail.html); }); // Editors mutate site-wide state (theme, structure, headings, banners), @@ -391,21 +417,22 @@ import "overlayscrollbars/overlayscrollbars.css"; }); function preload() { - const urls = new Set(); + const urls = new Map(); // cache key -> URL, deduped (hashes collapse) for (const a of document.querySelectorAll( '#nav a[href^="/"], #sidebar a[href^="/"], #main a[href^="/"]', )) { - urls.add(a.pathname); + const u = pageUrl(a.href); + urls.set(rawKey(u), u); } - for (const url of urls) { - if (pageCache.has(cacheKey(url))) continue; + for (const [key, u] of urls) { + if (pageCache.has(key)) continue; // x-pagerite-preload: idle cache warm-up, not a page view — the // server excludes these GETs from analytics (the navigation message // sent on actual navigation does the counting). - fetch(url, { headers: { "x-pagerite-preload": "1", ...pageHeaders } }) + fetch(u, { headers: { "x-pagerite-preload": "1" } }) .then((r) => (r.ok && (r.headers.get("content-type") || "").includes("text/html") ? r.text() : "")) - .then((html) => { if (html) pageCache.set(cacheKey(url), html); }) + .then((html) => { if (html) pageCache.set(key, html); }) .catch(() => {}); } } @@ -715,12 +742,12 @@ import "overlayscrollbars/overlayscrollbars.css"; teardownAnalytics(); let doc; let finalUrl = url; - const cached = !editing && pageCache.get(cacheKey(new URL(url, location.href).pathname)); + const cached = !editing && pageCache.get(cacheKey(url)); if (cached) { doc = new DOMParser().parseFromString(cached, "text/html"); } else { try { - const res = await fetch(url, { headers: pageHeaders }); + const res = await fetch(pageUrl(url)); const type = res.headers.get("content-type") || ""; if (!res.ok || !type.includes("text/html")) throw new Error("not a page"); // Reflect any redirect the server issued. @@ -728,15 +755,15 @@ import "overlayscrollbars/overlayscrollbars.css"; const html = await res.text(); // Populate the cache too, so returning here (back/forward, or a // self-link in the nav) is served from memory. - pageCache.set(cacheKey(new URL(finalUrl, location.href).pathname), html); + pageCache.set(cacheKey(finalUrl), html); doc = new DOMParser().parseFromString(html, "text/html"); } catch { - location.href = url; // fall back to a normal navigation + location.href = pageUrl(url); // fall back to a normal navigation return false; } } if (REGIONS.some((id) => !doc.getElementById(id))) { - location.href = url; + location.href = pageUrl(url); return false; } const doit = () => { @@ -816,7 +843,7 @@ import "overlayscrollbars/overlayscrollbars.css"; doit(); } currentPath = new URL(finalUrl, location.href).pathname; - if (push) history.pushState({ idx: ++historyIdx }, "", finalUrl); + if (push) history.pushState({ idx: ++historyIdx }, "", prettyUrl(finalUrl)); // The open editor follows the URL: retarget the per-page tabs to the // navigated-to page (unsaved text of the previous page is discarded — // the article it previewed into is gone). diff --git a/pagerite/app.py b/pagerite/app.py index 2845791..065d61c 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -388,13 +388,13 @@ class FileStore: file_store = FileStore(FILES_DIR) -def _render_html(kind: str, path: str, base_url: str, lang: str = i18n.ORIGINAL_LANGUAGE) -> str: +def _render_html(kind: str, path: str, base_url: str, lang: str = i18n.ORIGINAL_LANGUAGE, link_lang: str = "") -> str: """Render one of the generated pages (see _html_response).""" if kind == "page": # A selected language without an actual translation renders the # original (translation is None = English; see docs/localization.md). translation = i18n.get_translation(path, lang, data) if lang != i18n.ORIGINAL_LANGUAGE else None - return views.render_page(data.menu, data, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition, lang=lang, translation=translation) + return views.render_page(data.menu, data, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, base_url, transition=data.transition, lang=lang, translation=translation, link_lang=link_lang) if kind == "category": return views.render_category(data.menu, data, path, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition) if kind == "not-found": @@ -418,14 +418,17 @@ def _invalidate_pages() -> None: @lru_cache(maxsize=128) -def _cached_body(kind: str, path: str, base_url: str, zstd: bool, lang: str = i18n.ORIGINAL_LANGUAGE) -> bytes: +def _cached_body(kind: str, path: str, base_url: str, zstd: bool, lang: str = i18n.ORIGINAL_LANGUAGE, link_lang: str = "") -> bytes: """Rendered page body; cleared by _invalidate_pages on any content/settings change. base_url feeds the social meta URLs, zstd selects the stored encoding (both variants are cached rather than re-compressed) and lang the selected language (not the raw Accept-Language header, which would blow up the cache key space). + link_lang is the ?lang= override replicated onto the navigation links: + a query render and a header-selected render of the same language differ + in their links, so they are cached separately. """ - body = _render_html(kind, path, base_url, lang).encode() + body = _render_html(kind, path, base_url, lang, link_lang).encode() return _zstd.compress(body) if zstd else body @@ -437,6 +440,7 @@ def _html_response( headers: dict | None = None, etag: bool = False, lang: str = i18n.ORIGINAL_LANGUAGE, + link_lang: str = "", ) -> Response: """Response for a generated page, zstd-compressed when the client accepts it (no gzip fallback). @@ -458,11 +462,11 @@ def _html_response( # localhost (varying ports) fall back to the request's own base URL. base_url = SITE_URL or str(request.base_url).rstrip("/") if DEVMODE: - identity = _render_html(kind, path, base_url, lang).encode() + identity = _render_html(kind, path, base_url, lang, link_lang).encode() body = _zstd.compress(identity) if zstd else identity else: - identity = _cached_body(kind, path, base_url, False, lang) - body = _cached_body(kind, path, base_url, True, lang) if zstd else identity + identity = _cached_body(kind, path, base_url, False, lang, link_lang) + body = _cached_body(kind, path, base_url, True, lang, link_lang) if zstd else identity h = dict(headers or {}) # Content varies by language (Accept-Language selects a translation) # and by encoding; keep caches from mixing either representation. @@ -1750,18 +1754,24 @@ async def show_page(request: Request, path: str) -> Response: # Language selection (docs/localization.md): ?lang= wins when a # translation exists, else header logic. Analytics keep the raw # Accept-Language header regardless of the selection. + query_lang = request.query_params.get("lang") lang = i18n.select_language( - request.query_params.get("lang"), + query_lang, accept_language, lambda l: l in node.langs, ) + # A ?lang= override is replicated onto the page's navigation links + # (link_lang), so clicks and prefetches stay in the chosen language. + # Query and header-selected renders of the same language differ in + # their links, so link_lang is part of the ETag and body cache key. + link_lang = i18n.base_tag(query_lang or "") # no-cache forbids serving a stored page without revalidation # (browsers would otherwise cache heuristically and serve stale # pages, e.g. after a theme change). In-session speed instead comes # from pagerite.js's in-memory page cache (preload everything, never # fetch on navigation); the ETag just makes those one-time preload # fetches and any revalidation cheap. - etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}l{lang}"' + etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}l{lang}q{link_lang}"' if request.headers.get("if-none-match") == etag: return Response(status_code=304) if _is_trackable_path(path): @@ -1777,6 +1787,7 @@ async def show_page(request: Request, path: str) -> Response: "cache-control": "no-cache", }, lang=lang, + link_lang=link_lang, ) if node is not None and node.published and node.chunks is None: # Category label without a landing page: placeholder with the pen diff --git a/pagerite/data.py b/pagerite/data.py index d7978b0..138cbc0 100644 --- a/pagerite/data.py +++ b/pagerite/data.py @@ -51,8 +51,8 @@ class Node(msgspec.Struct, omit_defaults=True): no_trans: dict[bytes, bool] = {} #: Languages this article is available in (besides its primary #: language). Presence-keys, value always True — the availability - #: index for rendering, language selection and hreflang alternates; - #: maintained by whoever writes translation data (docs/migrate.md). + #: index for rendering and language selection; maintained by whoever + #: writes translation data (docs/migrate.md). langs: dict[str, bool] = {} #: Raw HTML for the header banner (img, styled div, canvas+script...), #: rendered after the banner design's artwork so author code always diff --git a/pagerite/i18n.py b/pagerite/i18n.py index d239bb0..feb42a2 100644 --- a/pagerite/i18n.py +++ b/pagerite/i18n.py @@ -178,12 +178,3 @@ def get_translation(path: str, lang: str, data: Data) -> Translation | None: markdown=hybrid_markdown(data, node, path, lang), titles=title_map(data, lang), ) - - -def available_languages(path: str, data: Data) -> list[str]: - """Languages the page at ``path`` is available in (besides the - original): the ``node.langs`` index, maintained by the translation - writers. Drives language selection and hreflang alternate links.""" - chain = resolve(data.menu, path) - node = chain[-1] if chain else None - return sorted(node.langs) if node else [] diff --git a/pagerite/views.py b/pagerite/views.py index d4f4af1..e97dea9 100644 --- a/pagerite/views.py +++ b/pagerite/views.py @@ -312,6 +312,7 @@ def _layout( favicon: str = "", social: dict[str, str] | None = None, lang: str = i18n.ORIGINAL_LANGUAGE, + canonical: str = "", alternates: list[tuple[str, str]] = (), ) -> Template: """Page layout template with standard assets and ES-module scripts. @@ -338,24 +339,25 @@ def _layout( ``social`` maps meta keys to contents: ``og:*``/``article:*`` go out as property attributes, everything else (description, twitter:*) as name. - ``lang`` is the served language for ; ``alternates`` holds - (hreflang, href) pairs for the other language versions of the page, - emitted as (see docs/localization.md). + ``lang`` is the served language for . ``canonical`` and + ``alternates`` ((hreflang, href) pairs) are the page's language URLs + (see docs/localization.md), emitted right after the viewport and before + the social tags: canonical first, then the hreflang alternates. """ doc = Document(E.Title, lang=lang) # Responsive layout (see the 48rem breakpoint in pagerite.css) needs # the real device width, not the default 980px layout viewport. doc.meta(name="viewport", content="width=device-width, initial-scale=1") + if canonical: + doc.link(rel="canonical", href=canonical) + for hreflang, href in alternates: + doc.link(rel="alternate", hreflang=hreflang, href=href) for key, value in (social or {}).items(): if value: if key.startswith(("og:", "article:")): doc.meta(property=key, content=value) - elif key == "canonical": - doc.link(rel="canonical", href=value) else: doc.meta(name=key, content=value) - for hreflang, href in alternates: - doc.link(rel="alternate", hreflang=hreflang, href=href) # A custom favicon (from the site editor) is linked explicitly; without # one, browsers fall back to the build's /favicon.ico by convention. if favicon: @@ -453,13 +455,13 @@ def _layout( return Template(body) -def _brand_link(brand: str, brand_html: str = "") -> HTML: +def _brand_link(brand: str, brand_html: str = "", link_lang: str = "") -> HTML: """Header brand: custom HTML (in a #brand wrapper, rendered instead of the link) when configured, else the plain brand link; omitted entirely when neither is set.""" if brand_html.strip(): return HTML(str(E.div(HTML(brand_html), id="brand"))) - return HTML(str(E.a(brand, href="/", id="brand"))) if brand else HTML("") + return HTML(str(E.a(brand, href=_href("", link_lang), id="brand"))) if brand else HTML("") def _title(slug: str, node: Node, translation: Translation | None = None, path: str = "") -> str: @@ -473,9 +475,18 @@ def _title(slug: str, node: Node, translation: Translation | None = None, path: return node.title or prettify(slug) or "Home" +def _href(path: str, link_lang: str = "") -> str: + """Site-chrome link to a page: when the page was requested with a + ?lang= override the query is replicated onto the navigation links it + renders, so clicks and prefetches (which take the href as-is) stay in + the chosen language — even without JS (docs/localization.md).""" + return f"/{path}?lang={link_lang}" if link_lang else f"/{path}" + + def _nav_link( doc, menu: dict[str, Node], node: Node, path: str, current: str, ancestors_current: bool = True, translation: Translation | None = None, + link_lang: str = "", ) -> None: """Render one
  • linking the node. Category labels (no content of their own — chunks None, or an empty page as left by the site editor's @@ -486,9 +497,9 @@ def _nav_link( is_current = current == path or ( ancestors_current and path and current.startswith(f"{path}/") ) - href = f"/{path}" + href = _href(path, link_lang) if not node.chunks and (leaf := first_leaf(menu, path)) is not None: - href = f"/{leaf}" + href = _href(leaf, link_lang) doc.li.a( _title(path.rpartition("/")[2], node, translation, path), href=href, @@ -496,7 +507,7 @@ def _nav_link( ) -def nav_html(menu: dict[str, Node], current: str, translation: Translation | None = None) -> HTML: +def nav_html(menu: dict[str, Node], current: str, translation: Translation | None = None, link_lang: str = "") -> HTML: """Render the contents of the #nav element for the current path. Top-level items in menu order; the front page (slug "", href "/") @@ -507,11 +518,11 @@ def nav_html(menu: dict[str, Node], current: str, translation: Translation | Non with nav: for slug, node in sorted_nodes(menu): if node.published: - _nav_link(nav, menu, node, slug, current, translation=translation) + _nav_link(nav, menu, node, slug, current, translation=translation, link_lang=link_lang) return HTML(str(nav)) -def sidebar_html(menu: dict[str, Node], current: str, translation: Translation | None = None) -> HTML: +def sidebar_html(menu: dict[str, Node], current: str, translation: Translation | None = None, link_lang: str = "") -> HTML: """Render the #sidebar element for the current path (empty when none). The sidebar is the current main level section's sub-navigation: the @@ -545,20 +556,20 @@ def sidebar_html(menu: dict[str, Node], current: str, translation: Translation | nav = E.ul with nav: for slug, child in items: - _sidebar_item(nav, menu, child, f"{section}/{slug}", current, translation) + _sidebar_item(nav, menu, child, f"{section}/{slug}", current, translation, link_lang) return HTML(str(E.aside(nav, id="sidebar"))) -def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: str, translation: Translation | None = None) -> None: +def _sidebar_item(doc, menu: dict[str, Node], node: Node, path: str, current: str, translation: Translation | None = None, link_lang: str = "") -> None: """One sidebar
  • : the node link, with its published children as a nested list (third level and deeper, recursively).""" - _nav_link(doc, menu, node, path, current, ancestors_current=False, translation=translation) + _nav_link(doc, menu, node, path, current, ancestors_current=False, translation=translation, link_lang=link_lang) sub = [(s, c) for s, c in sorted_nodes(node.children) if c.published] if sub: # doc.li.a(...) above left the
  • open for nesting. with doc.ul: for slug, child in sub: - _sidebar_item(doc, menu, child, f"{path}/{slug}", current, translation) + _sidebar_item(doc, menu, child, f"{path}/{slug}", current, translation, link_lang) def first_leaf(menu: dict[str, Node], path: str) -> str | None: @@ -690,7 +701,7 @@ def banner_source(menu: dict[str, Node], path: str) -> str | None: return None -def page_content(menu: dict[str, Node], data: Data, path: str, translation: Translation | None = None) -> HTML: +def page_content(menu: dict[str, Node], data: Data, path: str, translation: Translation | None = None, link_lang: str = "") -> HTML: """Render the contents of the #main element for a page. A page with published children (a category page) lists them as cards @@ -715,11 +726,11 @@ def page_content(menu: dict[str, Node], data: Data, path: str, translation: Tran doc = E.article(class_="multicol") if rendered.multicol else E.article with doc: doc(HTML(rendered.html)) - _cards(doc, menu, data, node, path, translation) + _cards(doc, menu, data, node, path, translation, link_lang) return HTML(str(doc)) -def _cards(doc, menu: dict[str, Node], data: Data, node: Node, path: str, translation: Translation | None = None) -> None: +def _cards(doc, menu: dict[str, Node], data: Data, node: Node, path: str, translation: Translation | None = None, link_lang: str = "") -> None: """Card stacks of the node's published children (nothing when childless). One column per direct child, all in a single full-width row (the .wide @@ -745,7 +756,7 @@ def _cards(doc, menu: dict[str, Node], data: Data, node: Node, path: str, transl continue with doc.div(class_="stack"): for epath, enode in entries: - _card(doc, data, enode, epath, translation) + _card(doc, data, enode, epath, translation, link_lang) def _walk(node: Node, path: str): @@ -759,7 +770,7 @@ def _walk(node: Node, path: str): yield from _walk(child, f"{path}/{slug}") -def _card(doc, data: Data, node: Node, path: str, translation: Translation | None = None) -> None: +def _card(doc, data: Data, node: Node, path: str, translation: Translation | None = None, link_lang: str = "") -> None: """One card in a stack: cover + title, plus the description when the page has no image (its card shows a gradient cover instead). @@ -772,7 +783,7 @@ def _card(doc, data: Data, node: Node, path: str, translation: Translation | Non image, _ = _media(html) if not image: description = _description(html, 150) - with doc.a(href=f"/{path}", class_="card"): + with doc.a(href=_href(path, link_lang), class_="card"): if image: doc.span(class_="cover", style=f'background-image: url("{image}")') else: @@ -861,7 +872,6 @@ def _share_media(html: str, base_url: str) -> tuple[str, str]: def _social_meta( node: Node, path: str, title: str, html: str, brand: str, base_url: str, - lang: str = i18n.ORIGINAL_LANGUAGE, ) -> dict[str, str]: """Open Graph/Twitter/SEO meta tags for a content page. @@ -871,9 +881,6 @@ def _social_meta( representative figure. Absolute URLs are built from the request's base (social scrapers cannot use relative ones). - When a translation is served, the canonical URL includes the ?lang= - query: each language version is its own canonical (docs/localization.md). - ``twitter:image`` pins extension-less store links to the ``.webp`` variant: X only honors WebP via twitter:image (not og:image) and its scraper cannot be trusted to negotiate via Accept. @@ -886,7 +893,6 @@ def _social_meta( ) return { "description": text, - "canonical": f"{url}?lang={lang}" if url and lang != i18n.ORIGINAL_LANGUAGE else url, "og:type": "article", "og:title": title, "og:description": text, @@ -914,37 +920,48 @@ def render_page( transition: str = "cube", lang: str = i18n.ORIGINAL_LANGUAGE, translation: Translation | None = None, + link_lang: str = "", ) -> str: """Render a full HTML page for the slug path. ``lang``/``translation`` serve a translated version (see docs/localization.md): None translation = the English original. + ``link_lang`` is the ?lang= override the page was requested with, + replicated onto the navigation links so the language sticks. """ node = resolve(menu, path)[-1] if translation is None: lang = i18n.ORIGINAL_LANGUAGE title = _title(path.rpartition("/")[2], node, translation, path) - main = page_content(menu, data, path, translation) - social = _social_meta(node, path, title, str(main), brand, base_url, lang) - # hreflang alternates: every other language version (with ?lang=) plus - # x-default for the plain URL. Emitted only when translations exist. + main = page_content(menu, data, path, translation, link_lang) + social = _social_meta(node, path, title, str(main), brand, base_url) + # Canonical/hreflang URLs (docs/localization.md): the canonical names + # the actually served language — the plain URL for the original (for + # SEO the non-query URL means the article's language), ?lang= for a + # translation — regardless of how the language was arrived at (query + # or header). The alternates are site-wide, the same set on every + # page: the configured translate_langs (the translator works to fill + # them all in), x-default first (the plain, autodetecting URL), then + # every language explicitly, the default language included. + canonical = "" alternates = [] if base_url: - available = i18n.available_languages(path, data) - alternates = [ - (l, f"{base_url}/{path}?lang={l}") for l in available if l != lang - ] - if available: - alternates.append(("x-default", f"{base_url}/{path}")) + url = f"{base_url}/{path}" + canonical = url if lang == i18n.ORIGINAL_LANGUAGE else f"{url}?lang={lang}" + if data.translate_langs: + alternates = [("x-default", url)] + [ + (l, f"{url}?lang={l}") + for l in [i18n.ORIGINAL_LANGUAGE, *sorted(data.translate_langs)] + ] return str( _layout( *_page_assets(), custom_css, theme, banner_design(menu, path, theme), - transition, favicon, social, lang, alternates, + transition, favicon, social, lang, canonical, alternates, )( Title=f"{title} – {brand}" if brand else title, - Brand=_brand_link(brand, brand_html), - Nav=nav_html(menu, path, translation), - Sidebar=sidebar_html(menu, path, translation), + Brand=_brand_link(brand, brand_html, link_lang), + Nav=nav_html(menu, path, translation, link_lang), + Sidebar=sidebar_html(menu, path, translation, link_lang), Banner=banner_html(menu, path, theme), Main=main, ), -- 2.55.0 From af76277d68117de78f8345061bc7beba222c04f8 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 17:48:54 +0000 Subject: [PATCH 18/31] Category pages now localized. --- docs/localization.md | 45 +++++- frontend/src/PageEditor.vue | 278 ++++++++++++++++++++++++++++++++---- pagerite/app.py | 106 +++++++++++--- pagerite/i18n.py | 11 ++ pagerite/views.py | 50 ++++--- pyproject.toml | 2 +- 6 files changed, 420 insertions(+), 72 deletions(-) diff --git a/docs/localization.md b/docs/localization.md index 0e20c3c..4fc231d 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -74,6 +74,14 @@ Region tags normalize to their base subtag (`fi-FI` → `fi`). - Navigation/sidebar titles come from the translation's title map, with per-node fallback to the original title (a partially translated tree must still render). +- Category placeholder pages (the 404s for content-less labels) select a + language like content pages, but over the **subtree's** combined + availability (`subtree_languages`) — they have no chunks of their own; + the heading, navigation and card text localize from the title map and + the target articles' translations. +- Card descriptions and cover picks run on the target article's hybrid + Markdown where that page is available in the served language, with + per-card fallback to the original. - Fixed UI strings ("Not Found" etc.) and the editor UI stay English for now. - The markdown typographer (SmartyPants) is English-centric; per-language typographer options are a possible follow-up, not blocking. @@ -211,13 +219,36 @@ def get_translation(path, lang, data) -> Translation | None: ### Editor flow -- `GET` of page Markdown for editing with a `lang` parameter returns the - hybrid (not the raw original) when the article has that language. -- `PUT`/WS save with `lang` does **not** touch `node.chunks`; it diffs - against the hybrid that was served and appends a `Patch`. (Serve a hybrid - generation token with the editor payload so a save based on a stale hybrid - can be rebased or rejected — simplest: recompute the diff against the - *current* hybrid and accept best-effort, matching the patch philosophy.) +The page editor has a language picker (flag + name; the same +country-flag-icons set as the analytics visitor cells) listing the primary +language and the union of the page's translations (`node.langs`) and the +site-wide `translate_langs`. It opens in the language the page was served +in (``). A note under the toolbar states the blast radius: +edits to the primary language re-chunk the original (invalidating the +affected translation fragments everywhere); edits to a translation stay +local to that language. + +- WS `open` with a `lang` returns the effective **hybrid** Markdown and + title for that language (ungated by `node.langs` — a language without + any fragments yet starts from the original text), plus the language + metadata (`lang`, `primary_lang`, `langs`, `translate_langs`). +- The editor keeps a **shadow copy** of the Markdown it opened. WS `save` + with `lang` sends it as `base`; the server diffs `base` → submitted text + (`make_patch`) and appends a `Patch`. Diffing against the shadow (rather + than the current hybrid) keeps hunks correct when the original or the + machine translation moved under an open editor; application against the + then-current hybrid stays best-effort per hunk, as designed. +- A changed **title** on a translated save becomes a fragment in + `Data.trans` keyed by the original title's chunk hash — the same storage + as machine title translations. An untouched title field (holding the + served translation) is not sent, so saving never freezes a stale machine + title into an override. +- Saving never deletes; a translation additionally cannot be emptied (that + would render as a blank page in that language). +- The live preview renders the version being edited, whichever language + the page itself was loaded in (the render is just the edited Markdown + + title). A translated save keeps that preview in place — re-fetching the + page would come back in the header-selected language. - Saving the primary-language version re-chunks the submitted Markdown and updates `Data.chunks` / `node.chunks` — only genuinely new text lands in the kanta change diff (see docs/migrate.md). diff --git a/frontend/src/PageEditor.vue b/frontend/src/PageEditor.vue index e85a282..1ed309e 100644 --- a/frontend/src/PageEditor.vue +++ b/frontend/src/PageEditor.vue @@ -11,14 +11,27 @@ // and refreshes the page regions in place — never a reload — so the editor // state (unsaved text included) also survives closing the shell. The editor // always follows the URL: navigating away retargets it to the new page, -// stashing unsaved text per path (unsavedStash) so returning to the page -// restores the working draft; stashes clear on save and on real reload. -import { onActivated, onMounted, onUnmounted, ref, watch } from 'vue' +// stashing unsaved text per path and language (stashes) so returning to the +// page restores the working draft; stashes clear on save and on real reload. +// +// Languages: the editor starts in the language the page was served in and +// the toolbar picker (flags, like the analytics visitor cells) switches +// between the primary language and its translations. A translation is +// edited as its effective (hybrid) Markdown; the hybrid the session +// started from is kept as a shadow copy (shadowBase) and sent along at +// save time, so the server diffs the user's changes only and stores them +// as a patch — edits to a translation never touch the original, while +// edits to the primary language re-chunk the original (and thereby +// invalidate the affected translation fragments). The live preview always +// renders the version being edited, whichever language the page itself +// was loaded in. +import { computed, onActivated, onMounted, onUnmounted, ref, watch } from 'vue' import { EditorView, basicSetup } from 'codemirror' import { EditorState } from '@codemirror/state' import { keymap } from '@codemirror/view' import { indentWithTab } from '@codemirror/commands' import { markdown } from '@codemirror/lang-markdown' +import * as flagSvgs from 'country-flag-icons/string/3x2' import { cmHighlight, cmTheme } from './cmtheme' import { dropPageCache, loadPlain } from './swapdoc' @@ -34,6 +47,20 @@ const saveError = ref('') const editorEl = ref(null) const fileInput = ref(null) +// The language being edited: "" = the primary language. Starts as the +// language this page was served in (); the server normalizes +// the primary to "" in its doc reply. +const lang = ref(document.documentElement.lang || '') +const primaryLang = ref('en') +const pageLangs = ref([]) // translations this page has +const siteLangs = ref([]) // site-wide configured target languages +const langPickerOpen = ref(false) +// The shadow copy: the Markdown this editing session started from, sent as +// "base" on translated saves so the server diffs the user's changes only. +let shadowBase = '' +let titleTouched = false +let docLoaded = false + let ws = null let view = null let savedResolve = null @@ -63,6 +90,55 @@ function normPath(p) { return p.trim().replace(/^\/+|\/+$/g, '') } +// --- Language picker ------------------------------------------------------- +// Flag icons from the same country-flag-icons set the analytics visitor +// cells use, resolved from the language tag's most likely region. +const displayNames = new Intl.DisplayNames(['en'], { type: 'language' }) + +function langName(tag) { + try { + return displayNames.of(tag) || tag + } catch { + return tag + } +} + +function flagFor(tag) { + try { + return flagSvgs[new Intl.Locale(tag).maximize().region] || '' + } catch { + return '' + } +} + +// The picker's options: the primary language first, then the union of the +// page's translations and the site-wide configured targets, sorted. +const langOptions = computed(() => { + const others = [...new Set([...siteLangs.value, ...pageLangs.value])] + .filter((l) => l && l !== primaryLang.value) + .sort() + return [primaryLang.value, ...others].map((code) => ({ + tag: code === primaryLang.value ? '' : code, + code, + name: langName(code), + flag: flagFor(code), + primary: code === primaryLang.value, + })) +}) + +const currentLang = computed( + () => langOptions.value.find((o) => o.tag === lang.value) + ?? { tag: '', code: lang.value || primaryLang.value, name: langName(lang.value || primaryLang.value), flag: flagFor(lang.value || primaryLang.value), primary: !lang.value }, +) + +function switchLang(tag) { + tag = tag || '' + if (tag === lang.value || !view) return + stashCurrent() + lang.value = tag + send({ type: 'open', path: path.value, lang: tag }) +} + function pageLabel() { return title.value.trim() || ('/' + (path.value || '')) } @@ -96,11 +172,17 @@ function save() { // never moves the page. const markdown = view.state.doc.toString() if (markdown.trim() === '') { + if (lang.value) { + // Emptying a translation would render it as a blank page; deleting + // pages is a primary-language action. + saveError.value = '⚠️ a translation cannot be emptied' + return Promise.resolve() + } // Empty text means delete — an explicit choice made here, in the page // editor; the save APIs (REST PUT / WS save) never delete on empty. return fetch(`/_api/pages/${path.value}`, { method: 'DELETE' }).then((res) => { saveError.value = res.ok ? '' : '⚠️ changes could not be saved' - if (res.ok) unsavedStash.delete(path.value) + if (res.ok) stashes.delete(stashKey(path.value, lang.value)) }) } const msg = { @@ -110,6 +192,15 @@ function save() { markdown, published: published.value, } + if (lang.value) { + msg.lang = lang.value + // The shadow copy this session started from: the server diffs base → + // markdown and stores only the user's changes as a patch. + msg.base = shadowBase + // An untouched title field is not sent: it holds the served + // translation, which a save must not freeze into an override fragment. + if (!titleTouched) delete msg.title + } pendingSave = msg send(msg) return new Promise((resolve) => { savedResolve = resolve }) @@ -120,9 +211,12 @@ async function saveAndRefresh() { dirty.value = false // Refresh the page regions from the server so nav/sidebar changes apply // (never a reload: the editor keeps its state). Drop the prefetch cache - // first: heading/title changes affect navigation on every page. + // first: heading/title changes affect navigation on every page. A + // translation save keeps the preview as-is — it already shows the saved + // text, and loadPlain would swap the article to the header-selected + // language's render. dropPageCache() - loadPlain(path.value) + if (!lang.value) loadPlain(path.value) } function close() { @@ -572,18 +666,27 @@ function insertTable(cols, rows) { view.focus() } -// Unsaved edits survive navigation within the session: leaving a page -// stashes its working text here, returning restores it (the server doc -// still arrives, for title/published and as the base underneath). -// Entries clear on save and on real reload (the shell is in-memory only). -const unsavedStash = new Map() +// Unsaved edits survive navigation within the session: leaving a page (or +// switching the language) stashes its working text and shadow base here, +// returning restores them (the server doc still arrives, for +// title/published and language metadata). Entries clear on save and on +// real reload (the shell is in-memory only). +const stashes = new Map() +const stashKey = (p, l) => `${p}|${l}` + +function stashCurrent() { + if (dirty.value && path.value) { + stashes.set(stashKey(path.value, lang.value), { + text: view.state.doc.toString(), + base: shadowBase, + }) + } +} function openPath(p) { - if (dirty.value && path.value && p !== path.value) { - unsavedStash.set(path.value, view.state.doc.toString()) - } + if (p !== path.value) stashCurrent() path.value = p - send({ type: 'open', path: p }) + send({ type: 'open', path: p, lang: lang.value }) } function setDocument(text, preserveSelection = false) { @@ -624,13 +727,25 @@ function previewIntoArticle(html, multicol) { function onMessage(ev) { const msg = JSON.parse(ev.data) - if (msg.type === 'doc' && msg.path === path.value) { + // The doc must answer the current path and language; before the first + // doc the language is not yet server-normalized (the primary arrives as + // "" while the picker may have started from an explicit code), so the + // first doc is accepted on path alone and adopts the echoed language. + if (msg.type === 'doc' && msg.path === path.value + && (!docLoaded || (msg.lang || '') === lang.value)) { + docLoaded = true title.value = msg.title published.value = msg.published + primaryLang.value = msg.primary_lang || 'en' + pageLangs.value = msg.langs || [] + siteLangs.value = msg.translate_langs || [] + lang.value = msg.lang || '' + titleTouched = false // Restore stashed unsaved edits over the server doc when returning // to a page left dirty. - const stashed = unsavedStash.get(msg.path) - setDocument(stashed ?? msg.markdown) + const stashed = stashes.get(stashKey(msg.path, lang.value)) + setDocument(stashed ? stashed.text : msg.markdown) + shadowBase = stashed ? stashed.base : msg.markdown dirty.value = stashed != null requestRender() // A section pen's target line survives the open/path-switch here. @@ -638,12 +753,20 @@ function onMessage(ev) { } else if (msg.type === 'html' && msg.path === path.value) { previewIntoArticle(msg.html, msg.multicol) } else if (msg.type === 'saved') { - saveError.value = '' - pendingSave = null - dirty.value = false - unsavedStash.delete(path.value) - savedResolve?.() - savedResolve = null + // A late "saved" for a save sent before a path/language switch must not + // clear the current session's state. + if (pendingSave && pendingSave.path === path.value + && (pendingSave.lang || '') === lang.value) { + saveError.value = '' + // The saved text becomes the shadow base for further saves. + shadowBase = pendingSave.markdown + pendingSave = null + dirty.value = false + titleTouched = false + stashes.delete(stashKey(path.value, lang.value)) + savedResolve?.() + savedResolve = null + } } else if (msg.type === 'error') { saveError.value = '⚠️ changes could not be saved' } @@ -929,11 +1052,34 @@ onUnmounted(() => {