From 10c4c9c4f966f232246feba4f653cbb140500e5c Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 2 Sep 2026 02:16:28 +0000 Subject: [PATCH] Translator service API: GET/POST /_api/translate/{lang} --- docs/localization.md | 7 ++- docs/migrate.md | 7 ++- pagerite/app.py | 121 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 4 deletions(-) diff --git a/docs/localization.md b/docs/localization.md index 7f2cf1f..1964cd2 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -213,8 +213,11 @@ def get_translation(path, lang, data) -> Translation | None: ### Explicitly out of scope for phase 2 - The machine translation itself: chunking output goes in, translated chunks - come back. A background job writes `trans` entries; this doc only defines - the storage key (`chunk_hash:lang`) and the merge semantics. + come back. The **service API exists** — `GET /_api/translate/{lang}` lists + pending items (`{"key", "text", "path", "kind"}`, key = base64 chunk hash), + `POST /_api/translate/{lang}` stores a batch (`{"items": [{key, text}]}`) + into `trans` and maintains `node.langs`; an external service does the + actual translating (gated by the /_api forward-auth like everything else). - Garbage collection of orphaned chunks/translations (see docs/migrate.md). - sitemap.xml per-language entries; translated UI chrome; per-language typographer options; multi-locale date/number formatting. diff --git a/docs/migrate.md b/docs/migrate.md index d5889c1..865551b 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -91,7 +91,12 @@ alternate links never enumerate chunks. It is written by whoever writes translation data, in the same transaction: - **Translator job:** after writing `trans[h][lang]` entries for an - article's chunks (or its title), set `node.langs[lang] = True`. + article's chunks (or its title), set `node.langs[lang] = True`. The + translation service API does both: `GET /_api/translate/{lang}` lists + pending items (titles + translatable chunks lacking an entry, deduped by + hash), `POST /_api/translate/{lang}` stores a batch into `trans`, sets + `langs` on every article that gained an entry and invalidates the page + cache — all in one transaction. - **Translated-view save:** appending the first patch for `f"{path}:{lang}"` sets `node.langs[lang] = True` (patches alone make the version exist). - **Removals:** deleting a patch or GC'ing translations re-derives the key: diff --git a/pagerite/app.py b/pagerite/app.py index 31155f0..852db30 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -13,6 +13,7 @@ walking the tree (``resolve``), moves are slot detach/attach """ import asyncio +import base64 import gzip import ipaddress import mimetypes @@ -49,7 +50,7 @@ from zstandard import ZstdCompressor from pagerite import analytics, i18n, seed, views from pagerite.__main__ import DEVMODE -from pagerite.chunks import store_chunks +from pagerite.chunks import chunk_key, needs_translation, store_chunks from pagerite.data import ( Data, Node, @@ -965,6 +966,124 @@ async def delete_page(path: str) -> None: _invalidate_pages() +def _check_translate_lang(lang: str) -> str: + """Normalize a ``/_api/translate`` language parameter; translations never + target the original language (its text lives in the chunk store, and an + "en" langs key would advertise a bogus ?lang=en alternate).""" + tag = i18n.base_tag(lang) + if not tag or tag == i18n.ORIGINAL_LANGUAGE: + raise HTTPException(400, "bad target language") + return tag + + +@app.get("/_api/translate/{lang}") +async def translate_pending(lang: str) -> list[dict]: + """Pending translation items for ``lang`` (translation service API). + + Every page node (published or not) contributes its title and each of + its chunks that needs translation (``needs_translation``), is not + editor-flagged no-translate (``node.no_trans``) and has no ``trans`` + entry for ``lang`` yet. Each item is + ``{"key", "text", "path", "kind"}``: ``key`` is the base64 of the + 9-byte chunk hash — the handle the service translates against and + POSTs back; ``text`` the original Markdown (or title); ``path`` the + article it came from (no leading slash); ``kind`` "chunk" or "title". + Items are deduped by key: content-addressed text (shared paragraphs, + repeated titles) is translated once, whichever page it first came from. + """ + lang = _check_translate_lang(lang) + items = [] + seen = set() + + def emit(key: bytes, text: str, path: str, kind: str) -> None: + if key in seen or lang in data.trans.get(key, {}): + return + seen.add(key) + items.append({ + "key": base64.b64encode(key).decode(), + "text": text, + "path": path, + "kind": kind, + }) + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.chunks is not None: + if node.title: + emit(chunk_key(node.title), node.title, path, "title") + for h in node.chunks: + text = data.chunks.get(h) + if ( + text is not None + and h not in node.no_trans + and needs_translation(text) + ): + emit(h, text, path, "chunk") + walk(node.children, path) + + walk(data.menu, "") + return items + + +class TranslationItemIn(BaseModel): + """One translated fragment submitted by the translation service.""" + + key: str # base64 of the 9-byte chunk hash, as issued by the GET + text: str # the translated Markdown (or title) + + +class TranslationsIn(BaseModel): + """Batch of translations for one language.""" + + items: list[TranslationItemIn] + + +@app.post("/_api/translate/{lang}") +async def translate_submit(lang: str, body: TranslationsIn) -> dict: + """Store a batch of machine translations for ``lang``, one transaction. + + Keys are the base64 chunk hashes issued by the GET; each text lands in + ``trans[key][lang]``. Unknown keys are stored anyway (unreferenced + hashes are never read, and the content may simply have moved on since + the GET); duplicates within a batch overwrite, last wins; a malformed + base64 key rejects the whole batch with 400. Every article that gained + at least one entry gets ``node.langs[lang]`` set (the availability + index), and cached pages are invalidated. Returns the stored count and + the paths of the articles that gained the language. + """ + lang = _check_translate_lang(lang) + try: + entries = [ + (base64.b64decode(item.key, validate=True), item.text) + for item in body.items + ] + except ValueError: + raise HTTPException(400, "malformed base64 key") from None + with kanta.transaction("submit translations", extra=lang): + stored = set() + for key, text in entries: + data.trans.setdefault(key, {})[lang] = text + stored.add(key) + pages = [] + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.chunks is not None: + keys = set(node.chunks) + if node.title: + keys.add(chunk_key(node.title)) + if keys & stored: + node.langs[lang] = True + pages.append(path) + walk(node.children, path) + + walk(data.menu, "") + _invalidate_pages() + return {"stored": len(entries), "pages": pages} + + _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")