Translator service API: GET/POST /_api/translate/{lang}

docs/localization.md

@@ -213,8 +213,11 @@ def get_translation(path, lang, data) -> Translation | None:
 ### Explicitly out of scope for phase 2
 
 - The machine translation itself: chunking output goes in, translated chunks
-  come back. A background job writes `trans` entries; this doc only defines
-  the storage key (`chunk_hash:lang`) and the merge semantics.
+  come back. The **service API exists** — `GET /_api/translate/{lang}` lists
+  pending items (`{"key", "text", "path", "kind"}`, key = base64 chunk hash),
+  `POST /_api/translate/{lang}` stores a batch (`{"items": [{key, text}]}`)
+  into `trans` and maintains `node.langs`; an external service does the
+  actual translating (gated by the /_api forward-auth like everything else).
 - Garbage collection of orphaned chunks/translations (see docs/migrate.md).
 - sitemap.xml per-language entries; translated UI chrome; per-language
   typographer options; multi-locale date/number formatting.

docs/migrate.md

@@ -91,7 +91,12 @@ alternate links never enumerate chunks. It is written by whoever writes
 translation data, in the same transaction:
 
 - **Translator job:** after writing `trans[h][lang]` entries for an
-  article's chunks (or its title), set `node.langs[lang] = True`.
+  article's chunks (or its title), set `node.langs[lang] = True`. The
+  translation service API does both: `GET /_api/translate/{lang}` lists
+  pending items (titles + translatable chunks lacking an entry, deduped by
+  hash), `POST /_api/translate/{lang}` stores a batch into `trans`, sets
+  `langs` on every article that gained an entry and invalidates the page
+  cache — all in one transaction.
 - **Translated-view save:** appending the first patch for `f"{path}:{lang}"`
   sets `node.langs[lang] = True` (patches alone make the version exist).
 - **Removals:** deleting a patch or GC'ing translations re-derives the key:

pagerite/app.py

@@ -13,6 +13,7 @@ walking the tree (``resolve``), moves are slot detach/attach
 """
 
 import asyncio
+import base64
 import gzip
 import ipaddress
 import mimetypes
@@ -49,7 +50,7 @@ from zstandard import ZstdCompressor
 
 from pagerite import analytics, i18n, seed, views
 from pagerite.__main__ import DEVMODE
-from pagerite.chunks import store_chunks
+from pagerite.chunks import chunk_key, needs_translation, store_chunks
 from pagerite.data import (
     Data,
     Node,
@@ -965,6 +966,124 @@ async def delete_page(path: str) -> None:
         _invalidate_pages()
 
 
+def _check_translate_lang(lang: str) -> str:
+    """Normalize a ``/_api/translate`` language parameter; translations never
+    target the original language (its text lives in the chunk store, and an
+    "en" langs key would advertise a bogus ?lang=en alternate)."""
+    tag = i18n.base_tag(lang)
+    if not tag or tag == i18n.ORIGINAL_LANGUAGE:
+        raise HTTPException(400, "bad target language")
+    return tag
+
+
+@app.get("/_api/translate/{lang}")
+async def translate_pending(lang: str) -> list[dict]:
+    """Pending translation items for ``lang`` (translation service API).
+
+    Every page node (published or not) contributes its title and each of
+    its chunks that needs translation (``needs_translation``), is not
+    editor-flagged no-translate (``node.no_trans``) and has no ``trans``
+    entry for ``lang`` yet. Each item is
+    ``{"key", "text", "path", "kind"}``: ``key`` is the base64 of the
+    9-byte chunk hash — the handle the service translates against and
+    POSTs back; ``text`` the original Markdown (or title); ``path`` the
+    article it came from (no leading slash); ``kind`` "chunk" or "title".
+    Items are deduped by key: content-addressed text (shared paragraphs,
+    repeated titles) is translated once, whichever page it first came from.
+    """
+    lang = _check_translate_lang(lang)
+    items = []
+    seen = set()
+
+    def emit(key: bytes, text: str, path: str, kind: str) -> None:
+        if key in seen or lang in data.trans.get(key, {}):
+            return
+        seen.add(key)
+        items.append({
+            "key": base64.b64encode(key).decode(),
+            "text": text,
+            "path": path,
+            "kind": kind,
+        })
+
+    def walk(nodes: dict[str, Node], prefix: str) -> None:
+        for slug, node in sorted_nodes(nodes):
+            path = f"{prefix}/{slug}" if prefix else slug
+            if node.chunks is not None:
+                if node.title:
+                    emit(chunk_key(node.title), node.title, path, "title")
+                for h in node.chunks:
+                    text = data.chunks.get(h)
+                    if (
+                        text is not None
+                        and h not in node.no_trans
+                        and needs_translation(text)
+                    ):
+                        emit(h, text, path, "chunk")
+            walk(node.children, path)
+
+    walk(data.menu, "")
+    return items
+
+
+class TranslationItemIn(BaseModel):
+    """One translated fragment submitted by the translation service."""
+
+    key: str  # base64 of the 9-byte chunk hash, as issued by the GET
+    text: str  # the translated Markdown (or title)
+
+
+class TranslationsIn(BaseModel):
+    """Batch of translations for one language."""
+
+    items: list[TranslationItemIn]
+
+
+@app.post("/_api/translate/{lang}")
+async def translate_submit(lang: str, body: TranslationsIn) -> dict:
+    """Store a batch of machine translations for ``lang``, one transaction.
+
+    Keys are the base64 chunk hashes issued by the GET; each text lands in
+    ``trans[key][lang]``. Unknown keys are stored anyway (unreferenced
+    hashes are never read, and the content may simply have moved on since
+    the GET); duplicates within a batch overwrite, last wins; a malformed
+    base64 key rejects the whole batch with 400. Every article that gained
+    at least one entry gets ``node.langs[lang]`` set (the availability
+    index), and cached pages are invalidated. Returns the stored count and
+    the paths of the articles that gained the language.
+    """
+    lang = _check_translate_lang(lang)
+    try:
+        entries = [
+            (base64.b64decode(item.key, validate=True), item.text)
+            for item in body.items
+        ]
+    except ValueError:
+        raise HTTPException(400, "malformed base64 key") from None
+    with kanta.transaction("submit translations", extra=lang):
+        stored = set()
+        for key, text in entries:
+            data.trans.setdefault(key, {})[lang] = text
+            stored.add(key)
+        pages = []
+
+        def walk(nodes: dict[str, Node], prefix: str) -> None:
+            for slug, node in sorted_nodes(nodes):
+                path = f"{prefix}/{slug}" if prefix else slug
+                if node.chunks is not None:
+                    keys = set(node.chunks)
+                    if node.title:
+                        keys.add(chunk_key(node.title))
+                    if keys & stored:
+                        node.langs[lang] = True
+                        pages.append(path)
+                walk(node.children, path)
+
+        walk(data.menu, "")
+        _invalidate_pages()
+    return {"stored": len(entries), "pages": pages}
+
+
 _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")