From 69962df0c2611c13be3252b27c904969beb4ed98 Mon Sep 17 00:00:00 2001
From: Leo Vasanko <git@vasanko.com>
Date: Wed, 2 Sep 2026 03:03:44 +0000
Subject: [PATCH] Translator WS protocol structs + pending/store core,
 Data.translate_key

---
 pagerite/data.py      |   4 ++
 pagerite/translate.py | 126 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 130 insertions(+)
 create mode 100644 pagerite/translate.py

diff --git a/pagerite/data.py b/pagerite/data.py
index 01c72bb..f7f0f3e 100644
--- a/pagerite/data.py
+++ b/pagerite/data.py
@@ -101,6 +101,10 @@ class Data(msgspec.Struct):
     #: linked as <link rel="icon"> on every page. Empty = the build's
     #: /favicon.ico.
     favicon: str = ""
+    #: API key gating the translator service WebSocket (/_translate/{key};
+    #: the external forward-auth does not cover that route). Generated
+    #: lazily at startup when empty (see lifespan in app.py).
+    translate_key: str = ""
     #: All original-language page text, content-addressed:
     #: chunk_key (9 bytes; base64 at the JSON level) -> Markdown chunk.
     #: Shared by every article.
diff --git a/pagerite/translate.py b/pagerite/translate.py
new file mode 100644
index 0000000..7328dd1
--- /dev/null
+++ b/pagerite/translate.py
@@ -0,0 +1,126 @@
+"""Translator service protocol and its transport-independent core.
+
+The external machine-translation service connects over WebSocket
+(``/_translate/<key>``, see app.py) and exchanges JSON frames decoded into
+the tagged msgspec structs below (``bytes`` fields ride as base64 — no
+manual encoding anywhere). This module holds the message structs plus the
+two computations shared by the WS handler: which fragments are pending for
+a language (``pending_items``) and storing a batch of results
+(``store_results``).
+"""
+
+import msgspec
+
+from pagerite.chunks import chunk_key, needs_translation
+from pagerite.data import Data, Node, sorted_nodes
+
+
+class Hello(msgspec.Struct, tag="hello"):
+    """Client greeting on connect: the target languages it handles."""
+
+    langs: list[str]
+
+
+class TransItem(msgspec.Struct):
+    """One fragment to translate: original Markdown (or a node title)."""
+
+    key: bytes  #: 9-byte chunk hash (base64 in the JSON frame)
+    text: str
+    path: str  #: article it came from ("" = front page), no leading slash
+    kind: str  #: "chunk" | "title"
+
+
+class Job(msgspec.Struct, tag="job"):
+    """Server push: pending items for one language."""
+
+    lang: str
+    items: list[TransItem]
+
+
+class TransResult(msgspec.Struct):
+    """One translated fragment."""
+
+    key: bytes
+    text: str
+
+
+class Result(msgspec.Struct, tag="result"):
+    """Client reply: a batch of translations for one language."""
+
+    lang: str
+    items: list[TransResult]
+
+
+#: Union of the client -> server frames (the "type" tag selects).
+ClientMsg = Hello | Result
+
+
+def pending_items(data: Data, lang: str) -> list[TransItem]:
+    """Fragments of the site still untranslated for ``lang``, deduped by key.
+
+    Every page node (published or not) contributes its title and each chunk
+    that needs translation (``needs_translation``), is not editor-flagged
+    no-translate (``node.no_trans``) and has no ``trans`` entry for ``lang``
+    yet. Content-addressed text (shared paragraphs, repeated titles) appears
+    once, under the first page in menu order that has it.
+    """
+    items: list[TransItem] = []
+    seen: set[bytes] = set()
+
+    def emit(key: bytes, text: str, path: str, kind: str) -> None:
+        if key in seen or lang in data.trans.get(key, {}):
+            return
+        seen.add(key)
+        items.append(TransItem(key=key, text=text, path=path, kind=kind))
+
+    def walk(nodes: dict[str, Node], prefix: str) -> None:
+        for slug, node in sorted_nodes(nodes):
+            path = f"{prefix}/{slug}" if prefix else slug
+            if node.chunks is not None:
+                if node.title:
+                    emit(chunk_key(node.title), node.title, path, "title")
+                for h in node.chunks:
+                    text = data.chunks.get(h)
+                    if (
+                        text is not None
+                        and h not in node.no_trans
+                        and needs_translation(text)
+                    ):
+                        emit(h, text, path, "chunk")
+            walk(node.children, path)
+
+    walk(data.menu, "")
+    return items
+
+
+def store_results(data: Data, lang: str, items: list[TransResult]) -> list[str]:
+    """Store a batch of machine translations for ``lang``; return the paths
+    of the articles that gained at least one entry.
+
+    Pure data operations: the caller wraps this in a kanta transaction and
+    invalidates pages. Unknown keys are stored anyway (unreferenced hashes
+    are never read, and the content may simply have moved on since the job
+    was pushed); duplicates within a batch overwrite, last wins. Every
+    article that gained an entry gets ``node.langs[lang]`` set (the
+    availability index, docs/migrate.md) — because chunks are
+    content-addressed, that includes pages merely sharing a fragment.
+    """
+    stored = {item.key for item in items}
+    for item in items:
+        data.trans.setdefault(item.key, {})[lang] = item.text
+    pages: list[str] = []
+
+    def walk(nodes: dict[str, Node], prefix: str) -> None:
+        for slug, node in sorted_nodes(nodes):
+            path = f"{prefix}/{slug}" if prefix else slug
+            if node.chunks is not None:
+                keys = set(node.chunks)
+                if node.title:
+                    keys.add(chunk_key(node.title))
+                if keys & stored:
+                    node.langs[lang] = True
+                    pages.append(path)
+            walk(node.children, path)
+
+    walk(data.menu, "")
+    return pages