From 6e9f73f83a547124b7d37cc82bdd1af251dabd46 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 01:34:13 +0000 Subject: [PATCH] Real translation storage: chunk lookup, user patches, title map --- pagerite/i18n.py | 136 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 18 deletions(-) diff --git a/pagerite/i18n.py b/pagerite/i18n.py index 126d8dd..6ae7802 100644 --- a/pagerite/i18n.py +++ b/pagerite/i18n.py @@ -1,14 +1,20 @@ -"""Localization: language selection and translation access. +"""Localization: language selection, translation storage and assembly. -See docs/localization.md. The database keeps only the original language -(English); translations of page Markdown and navigation titles are consumed -through get_translation(), with per-node fallback to the original titles. +See docs/localization.md and docs/migrate.md. The database holds the +original language as content-addressed chunks (``Data.chunks``); per +target language there are machine-translated fragments (``Data.trans``) +and user override patches (``Data.patches``), assembled into the served +Markdown at render time, with per-node fallback to the original titles. """ from collections.abc import Callable +from difflib import SequenceMatcher import msgspec +from pagerite.chunks import chunk_key, chunk_markdown, join_chunks +from pagerite.data import Data, Node, Patch, resolve + #: Language of the database originals (and the default ). ORIGINAL_LANGUAGE = "en" @@ -26,6 +32,11 @@ class Translation(msgspec.Struct, omit_defaults=True): titles: dict[str, str] = {} +def base_tag(tag: str) -> str: + """The lowercase base subtag of a language tag (fi-FI -> fi).""" + return tag.strip().lower().partition("-")[0] + + def parse_accept_language(header: str) -> list[str]: """Accept-Language header as an ordered, deduped list of base subtags. @@ -35,7 +46,7 @@ def parse_accept_language(header: str) -> list[str]: """ langs = [] for part in header.split(","): - tag = part.split(";", 1)[0].strip().lower().partition("-")[0] + tag = base_tag(part.split(";", 1)[0]) if tag and tag != "*" and tag not in langs: langs.append(tag) return langs @@ -58,7 +69,7 @@ def select_language( 4. Fall back to the original. """ if query_lang: - tag = query_lang.strip().lower().partition("-")[0] + tag = base_tag(query_lang) if tag == original or (tag and is_available(tag)): return tag langs = parse_accept_language(accept_language or "") @@ -70,20 +81,109 @@ def select_language( return original -def get_translation(path: str, lang: str) -> Translation | None: +def apply_patch(hybrid: str, patch: Patch) -> str: + """Apply one patch to the hybrid Markdown, best effort, each hunk + independently: a hunk whose search text no longer exists is stale and + silently skipped (docs/localization.md).""" + for search, replace in patch.hunks: + if search and search in hybrid: + hybrid = hybrid.replace(search, replace, 1) + return hybrid + + +def make_patch(base: str, edited: str) -> Patch: + """The minimal diff of ``edited`` against the served ``base`` hybrid as + (search, replace) hunks at block granularity (docs/localization.md). + + Blocks are the chunk_markdown split, so hunks align with translation + units and code fences never straddle a hunk boundary. Pure inserts + anchor on the preceding block (an empty search would never match); + inserts at the very top anchor on the first block. autojunk is off: + the diff must be deterministic, and pages are small. + """ + a, b = chunk_markdown(base), chunk_markdown(edited) + hunks: list[tuple[str, str]] = [] + for tag, i1, i2, j1, j2 in SequenceMatcher(None, a, b, autojunk=False).get_opcodes(): + if tag == "equal": + continue + search = "\n\n".join(a[i1:i2]) + replace = "\n\n".join(b[j1:j2]) + if tag == "insert": + if i1: + search = a[i1 - 1] + replace = f"{a[i1 - 1]}\n\n{replace}" + elif a: + search = a[0] + replace = f"{replace}\n\n{a[0]}" + # else: base is empty — the hunk is inert (empty search is + # skipped by apply_patch); saving a translation of an empty + # page records nothing applicable. + hunks.append((search, replace)) + return Patch(hunks=hunks) + + +def hybrid_markdown(data: Data, node: Node, path: str, lang: str) -> str: + """The served Markdown for ``lang``: per chunk the translation from + ``Data.trans``, unless missing or marked no-translate (fallback to the + original chunk), then the language's user patches applied in order. + + Not gated on ``node.langs`` (get_translation is the gated view): the + editor save path diffs against this even for a language's first patch. + """ + hybrid = join_chunks([ + data.chunks.get(h, "") + if h in node.no_trans + else data.trans.get(f"{h}:{lang}") or data.chunks.get(h, "") + for h in node.chunks or [] + ]) + for patch in data.patches.get(f"{path}:{lang}", []): + hybrid = apply_patch(hybrid, patch) + return hybrid + + +def title_map(data: Data, lang: str) -> dict[str, str]: + """path -> translated title for every node that has one. + + Titles are chunks too (docs/migrate.md): keyed by the hash of the + title text, so editing a title invalidates its translations. Nodes + without an entry fall back to their original title in views. + """ + titles = {} + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in nodes.items(): + path = f"{prefix}/{slug}" if prefix else slug + if node.title: + t = data.trans.get(f"{chunk_key(node.title)}:{lang}") + if t: + titles[path] = t + walk(node.children, path) + + walk(data.menu, "") + return titles + + +def get_translation(path: str, lang: str, data: Data) -> Translation | None: """The translation of the page at ``path`` for ``lang``, or None. - TODO: stub for the AI-translation track, which will provide the real - implementation (storage, generation, invalidation on source changes). - The CMS only consumes this function and available_languages(). + None when the page does not exist or is not available in ``lang``: + ``node.langs`` is the availability index (a stale key is benign — the + "translation" then just renders as the original). """ - return None + chain = resolve(data.menu, path) + node = chain[-1] if chain else None + if node is None or node.chunks is None or lang not in node.langs: + return None + return Translation( + markdown=hybrid_markdown(data, node, path, lang), + titles=title_map(data, lang), + ) -def available_languages(path: str) -> list[str]: - """Languages with a translation for the page at ``path`` (besides the - original). Drives language selection and hreflang alternate links. - - TODO: stub, together with get_translation(). - """ - return [] +def available_languages(path: str, data: Data) -> list[str]: + """Languages the page at ``path`` is available in (besides the + original): the ``node.langs`` index, maintained by the translation + writers. Drives language selection and hreflang alternate links.""" + chain = resolve(data.menu, path) + node = chain[-1] if chain else None + return sorted(node.langs) if node else []