Localization #1
+118
-18
@@ -1,14 +1,20 @@
|
||||
"""Localization: language selection and translation access.
|
||||
"""Localization: language selection, translation storage and assembly.
|
||||
|
||||
See docs/localization.md. The database keeps only the original language
|
||||
(English); translations of page Markdown and navigation titles are consumed
|
||||
through get_translation(), with per-node fallback to the original titles.
|
||||
See docs/localization.md and docs/migrate.md. The database holds the
|
||||
original language as content-addressed chunks (``Data.chunks``); per
|
||||
target language there are machine-translated fragments (``Data.trans``)
|
||||
and user override patches (``Data.patches``), assembled into the served
|
||||
Markdown at render time, with per-node fallback to the original titles.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
import msgspec
|
||||
|
||||
from pagerite.chunks import chunk_key, chunk_markdown, join_chunks
|
||||
from pagerite.data import Data, Node, Patch, resolve
|
||||
|
||||
#: Language of the database originals (and the default <html lang>).
|
||||
ORIGINAL_LANGUAGE = "en"
|
||||
|
||||
@@ -26,6 +32,11 @@ class Translation(msgspec.Struct, omit_defaults=True):
|
||||
titles: dict[str, str] = {}
|
||||
|
||||
|
||||
def base_tag(tag: str) -> str:
|
||||
"""The lowercase base subtag of a language tag (fi-FI -> fi)."""
|
||||
return tag.strip().lower().partition("-")[0]
|
||||
|
||||
|
||||
def parse_accept_language(header: str) -> list[str]:
|
||||
"""Accept-Language header as an ordered, deduped list of base subtags.
|
||||
|
||||
@@ -35,7 +46,7 @@ def parse_accept_language(header: str) -> list[str]:
|
||||
"""
|
||||
langs = []
|
||||
for part in header.split(","):
|
||||
tag = part.split(";", 1)[0].strip().lower().partition("-")[0]
|
||||
tag = base_tag(part.split(";", 1)[0])
|
||||
if tag and tag != "*" and tag not in langs:
|
||||
langs.append(tag)
|
||||
return langs
|
||||
@@ -58,7 +69,7 @@ def select_language(
|
||||
4. Fall back to the original.
|
||||
"""
|
||||
if query_lang:
|
||||
tag = query_lang.strip().lower().partition("-")[0]
|
||||
tag = base_tag(query_lang)
|
||||
if tag == original or (tag and is_available(tag)):
|
||||
return tag
|
||||
langs = parse_accept_language(accept_language or "")
|
||||
@@ -70,20 +81,109 @@ def select_language(
|
||||
return original
|
||||
|
||||
|
||||
def get_translation(path: str, lang: str) -> Translation | None:
|
||||
def apply_patch(hybrid: str, patch: Patch) -> str:
|
||||
"""Apply one patch to the hybrid Markdown, best effort, each hunk
|
||||
independently: a hunk whose search text no longer exists is stale and
|
||||
silently skipped (docs/localization.md)."""
|
||||
for search, replace in patch.hunks:
|
||||
if search and search in hybrid:
|
||||
hybrid = hybrid.replace(search, replace, 1)
|
||||
return hybrid
|
||||
|
||||
|
||||
def make_patch(base: str, edited: str) -> Patch:
|
||||
"""The minimal diff of ``edited`` against the served ``base`` hybrid as
|
||||
(search, replace) hunks at block granularity (docs/localization.md).
|
||||
|
||||
Blocks are the chunk_markdown split, so hunks align with translation
|
||||
units and code fences never straddle a hunk boundary. Pure inserts
|
||||
anchor on the preceding block (an empty search would never match);
|
||||
inserts at the very top anchor on the first block. autojunk is off:
|
||||
the diff must be deterministic, and pages are small.
|
||||
"""
|
||||
a, b = chunk_markdown(base), chunk_markdown(edited)
|
||||
hunks: list[tuple[str, str]] = []
|
||||
for tag, i1, i2, j1, j2 in SequenceMatcher(None, a, b, autojunk=False).get_opcodes():
|
||||
if tag == "equal":
|
||||
continue
|
||||
search = "\n\n".join(a[i1:i2])
|
||||
replace = "\n\n".join(b[j1:j2])
|
||||
if tag == "insert":
|
||||
if i1:
|
||||
search = a[i1 - 1]
|
||||
replace = f"{a[i1 - 1]}\n\n{replace}"
|
||||
elif a:
|
||||
search = a[0]
|
||||
replace = f"{replace}\n\n{a[0]}"
|
||||
# else: base is empty — the hunk is inert (empty search is
|
||||
# skipped by apply_patch); saving a translation of an empty
|
||||
# page records nothing applicable.
|
||||
hunks.append((search, replace))
|
||||
return Patch(hunks=hunks)
|
||||
|
||||
|
||||
def hybrid_markdown(data: Data, node: Node, path: str, lang: str) -> str:
|
||||
"""The served Markdown for ``lang``: per chunk the translation from
|
||||
``Data.trans``, unless missing or marked no-translate (fallback to the
|
||||
original chunk), then the language's user patches applied in order.
|
||||
|
||||
Not gated on ``node.langs`` (get_translation is the gated view): the
|
||||
editor save path diffs against this even for a language's first patch.
|
||||
"""
|
||||
hybrid = join_chunks([
|
||||
data.chunks.get(h, "")
|
||||
if h in node.no_trans
|
||||
else data.trans.get(f"{h}:{lang}") or data.chunks.get(h, "")
|
||||
for h in node.chunks or []
|
||||
])
|
||||
for patch in data.patches.get(f"{path}:{lang}", []):
|
||||
hybrid = apply_patch(hybrid, patch)
|
||||
return hybrid
|
||||
|
||||
|
||||
def title_map(data: Data, lang: str) -> dict[str, str]:
|
||||
"""path -> translated title for every node that has one.
|
||||
|
||||
Titles are chunks too (docs/migrate.md): keyed by the hash of the
|
||||
title text, so editing a title invalidates its translations. Nodes
|
||||
without an entry fall back to their original title in views.
|
||||
"""
|
||||
titles = {}
|
||||
|
||||
def walk(nodes: dict[str, Node], prefix: str) -> None:
|
||||
for slug, node in nodes.items():
|
||||
path = f"{prefix}/{slug}" if prefix else slug
|
||||
if node.title:
|
||||
t = data.trans.get(f"{chunk_key(node.title)}:{lang}")
|
||||
if t:
|
||||
titles[path] = t
|
||||
walk(node.children, path)
|
||||
|
||||
walk(data.menu, "")
|
||||
return titles
|
||||
|
||||
|
||||
def get_translation(path: str, lang: str, data: Data) -> Translation | None:
|
||||
"""The translation of the page at ``path`` for ``lang``, or None.
|
||||
|
||||
TODO: stub for the AI-translation track, which will provide the real
|
||||
implementation (storage, generation, invalidation on source changes).
|
||||
The CMS only consumes this function and available_languages().
|
||||
None when the page does not exist or is not available in ``lang``:
|
||||
``node.langs`` is the availability index (a stale key is benign — the
|
||||
"translation" then just renders as the original).
|
||||
"""
|
||||
return None
|
||||
chain = resolve(data.menu, path)
|
||||
node = chain[-1] if chain else None
|
||||
if node is None or node.chunks is None or lang not in node.langs:
|
||||
return None
|
||||
return Translation(
|
||||
markdown=hybrid_markdown(data, node, path, lang),
|
||||
titles=title_map(data, lang),
|
||||
)
|
||||
|
||||
|
||||
def available_languages(path: str) -> list[str]:
|
||||
"""Languages with a translation for the page at ``path`` (besides the
|
||||
original). Drives language selection and hreflang alternate links.
|
||||
|
||||
TODO: stub, together with get_translation().
|
||||
"""
|
||||
return []
|
||||
def available_languages(path: str, data: Data) -> list[str]:
|
||||
"""Languages the page at ``path`` is available in (besides the
|
||||
original): the ``node.langs`` index, maintained by the translation
|
||||
writers. Drives language selection and hreflang alternate links."""
|
||||
chain = resolve(data.menu, path)
|
||||
node = chain[-1] if chain else None
|
||||
return sorted(node.langs) if node else []
|
||||
|
||||
Reference in New Issue
Block a user