From 859d11491ddd30512797a21689cb80817b93e6f4 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Mon, 21 Sep 2026 01:07:54 +0000 Subject: [PATCH] article mode: inject the page title as # heading, jargon loanword rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the render would inject the page title as an h1 (markdown.has_h1 is False for the body), an article job's text carries the same '# {title}' line: the title translates in document context and the opening paragraphs see the heading (works both ways). The heading's pair in the decomposed result becomes the title fragment — heading text only, pure prose, never stored as a body chunk; a demoted or merged heading skips the title, which stays pending for a scoped title job. The job covers the title key so it is not double-dispatched afterwards. Prompt: prefer established technical loanwords with English roots over forced localizations (frontend -> frontti in Finnish, not etupääte). Verified offline (title extraction, demoted-heading skip, no-injection path) and live against ollama qwen3.8:27b: the standalone title job gave 'Alkuun pääseminen', the article job then overwrote it with the in-context 'Aloitus' matching the body's translated heading. --- docs/llm-translation.md | 7 +++- docs/localization.md | 10 ++++-- pagerite/translate.py | 75 +++++++++++++++++++++++++++++++-------- scripts/llm_translator.py | 3 +- 4 files changed, 76 insertions(+), 19 deletions(-) diff --git a/docs/llm-translation.md b/docs/llm-translation.md index e3e289c..6d77b57 100644 --- a/docs/llm-translation.md +++ b/docs/llm-translation.md @@ -125,7 +125,12 @@ Three job modes, in increasing granularity: Titles are jobs like any other in all modes (`kind="title"` keeps its article-opening context rule; in `markdown` mode a title crosses as -plain text, since it carries no markup by construction). +plain text, since it carries no markup by construction). Additionally, an +`article` job carries the page title injected as a `# {title}` line at +the top when the render would inject it (the body has no h1 of its own): +the title translates in document context and the opening paragraphs see +the heading. The heading's pair in the decomposed result becomes the +title fragment (heading text only, never stored as a body chunk). ### Dispatch and validation diff --git a/docs/localization.md b/docs/localization.md index 26ddf30..ac2e9bc 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -418,13 +418,19 @@ offerable to clients of another approach. connections and only while a page is *mostly* pending (a new article or a full refresh; steady-state edit follow-up stays scoped jobs). The job's key is the page's first chunk; `Job.texts` carries the full - original Markdown. The result is decomposed per chunk + original Markdown — with the page title injected as a `# {title}` line + at the top when the render would inject it (the body has no h1 of its + own), so the title translates in document context and the opening + paragraphs see the heading. The result is decomposed per chunk (`align_article`): non-translatable blocks (code fences, container fences, raw HTML — everything `needs_translation` rejects) must appear verbatim and in order and anchor the alignment; regions between anchors pair positionally, a region whose block count changed stores nothing (its chunks stay pending and fall back to scoped jobs), and a paired - block whose destinations/placeholders did not survive likewise. + block whose destinations/placeholders did not survive likewise. An + injected title heading's pair becomes the title fragment (heading text + only — never a body chunk; a demoted or merged heading simply skips it + and the title stays pending for a scoped title job). `scripts/llm_translator.py` is the reference markdown+article client (instruct LLMs via an OpenAI Chat Completions endpoint or ollama's native diff --git a/pagerite/translate.py b/pagerite/translate.py index 1ed9df2..c9607f5 100644 --- a/pagerite/translate.py +++ b/pagerite/translate.py @@ -42,6 +42,7 @@ from pagerite.chunks import ( needs_translation, ) from pagerite.data import Data, Node, node_markdown, resolve, sorted_nodes +from pagerite.markdown import has_h1 from pagerite.segments import Span, join, pure_prose, split logger = logging.getLogger(__name__) @@ -121,6 +122,11 @@ class Result(msgspec.Struct, tag="result"): #: Union of the client -> server frames (the "type" tag selects). ClientMsg = Hello | Result +#: A dispatchable offer: the job, its segment spans (segments mode), the +#: full source text, the (lang, key) pairs it covers, and the page title's +#: chunk key when an article job carries an injected title heading. +_Offer = tuple["Job", list[Span], str, set[tuple[str, bytes]], "bytes | None"] + #: Constructs a translation must preserve verbatim inside a prose block: #: link/image destinations and {...} placeholders (sorted multisets are #: compared, so additions and drops both fail validation). @@ -307,16 +313,21 @@ class _Connection: self.spans: list[Span] = [] self.original: str = "" # its full source text (splicing / alignment) self.kind: str = "" # "chunk" | "title" | "article" + #: Article jobs with an injected title heading: the page title's + #: chunk key (its translation is extracted from the result's first + #: block, never stored as a body chunk). + self.title_key: bytes | None = None - def take(self) -> tuple[str, str, str, list[Span]]: + def take(self) -> tuple[str, str, str, list[Span], bytes | None]: """Snapshot and clear the in-flight job's working state.""" - mode, kind, spans, original = self.mode, self.kind, self.spans, self.original + out = (self.mode, self.kind, self.spans, self.original, self.title_key) self.inflight = None self.items = set() self.mode = self.kind = "" self.spans = [] self.original = "" - return mode, kind, spans, original + self.title_key = None + return out class Dispatcher: @@ -370,11 +381,9 @@ class Dispatcher: return asyncio.create_task(self._dispatch()) - def _scoped_job( - self, item: TransItem, lang: str, mode: str - ) -> tuple[Job, list[Span], str, set[tuple[str, bytes]]] | None: + def _scoped_job(self, item: TransItem, lang: str, mode: str) -> _Offer | None: """A title/chunk job for one pending item, in segments or markdown - mode: (job, spans, original, covered (lang, key) pairs).""" + mode.""" if mode == "segments": spans, texts, contexts = split(item.text) if not texts: @@ -405,7 +414,13 @@ class Dispatcher: mode="markdown", contexts=contexts, ) - return job, spans if mode == "segments" else [], item.text, {(lang, item.key)} + return ( + job, + spans if mode == "segments" else [], + item.text, + {(lang, item.key)}, + None, + ) def _block_contexts(self, lang: str, item: TransItem) -> list[str]: """The previous and next block of the served hybrid around a pending @@ -439,10 +454,15 @@ class Dispatcher: def _article_job( self, lang: str, items: list[TransItem], inflight: set[tuple[str, bytes]] - ) -> tuple[Job, list[Span], str, set[tuple[str, bytes]]] | None: + ) -> _Offer | None: """A whole-page job for the first page that is mostly pending for ``lang`` (a whole new article or a full refresh; steady-state edits - stay scoped jobs). The job's key is the page's first chunk.""" + stay scoped jobs). The job's key is the page's first chunk. + + When the render would inject the page title as an h1 (the body has + none of its own), the job text carries the same ``# {title}`` line: + the title translates in document context, and the opening + paragraphs see the heading.""" by_path: dict[str, list[TransItem]] = {} for item in items: if item.kind == "chunk": @@ -470,15 +490,22 @@ class Dispatcher: if (lang, key, "article") in self.validation_failures: continue md = node_markdown(self.data, node) or "" + title_key = None + if node.title and not has_h1(md): + md = f"# {node.title}\n\n{md}" + title_key = chunk_key(node.title) + covered = {(lang, k) for k in pend} + if title_key is not None: + covered.add((lang, title_key)) job = Job( lang=lang, key=key, texts=[md], path=path, kind="article", mode="article" ) - return job, [], md, {(lang, k) for k in pend} + return job, [], md, covered, title_key return None def _pick( self, state: _Connection, langs: list[str], inflight: set[tuple[str, bytes]] - ) -> tuple[Job, list[Span], str, set[tuple[str, bytes]]] | None: + ) -> _Offer | None: """The next job for a free connection: titles before articles before chunks — across languages too, so every menu is named before any article body is worked on (a page's name is its most visible @@ -530,13 +557,14 @@ class Dispatcher: offer = self._pick(state, langs, inflight) if offer is None: continue - job, spans, original, items = offer + job, spans, original, items, title_key = offer state.inflight = (job.lang, job.key) # before the await: no double-assign state.mode = job.mode state.kind = job.kind state.spans = spans state.original = original state.items = items + state.title_key = title_key try: await ws.send_text(msgspec.json.encode(job).decode()) except Exception: # send failed: the receive loop cleans up @@ -550,6 +578,7 @@ class Dispatcher: original: str, spans: list[Span], texts: list[str], + title_key: bytes | None = None, ) -> list[TransResult] | None: """Validate a Result against its in-flight job and turn it into storable fragments; None when it fails validation (the caller skips @@ -563,6 +592,22 @@ class Dispatcher: pairs = align_article(original, texts[0]) if len(texts) == 1 else None if not pairs: return None + if title_key is not None: + # The job carried an injected "# {title}" heading: its pair + # becomes the title fragment (heading text only, never a body + # chunk). A demoted/merged heading just skips the title — it + # stays pending for a scoped title job. + heading = chunk_key(chunk_markdown(original)[0]) + title = "" + kept = [] + for k, t in pairs: + if k == heading and not title: + m = re.fullmatch(r"# (.+)", t) + if m and pure_prose(m.group(1)): + title = m.group(1) + continue + kept.append((k, t)) + pairs = ([(title_key, title)] if title else []) + kept return [TransResult(key=k, text=t) for k, t in pairs] async def handle_ws(self, ws: WebSocket, clientkey: str) -> None: @@ -617,9 +662,9 @@ class Dispatcher: ): await ws.close(code=1002) return - mode, kind, spans, original = state.take() + mode, kind, spans, original, title_key = state.take() results = self._results( - mode, kind, msg.key, original, spans, msg.texts + mode, kind, msg.key, original, spans, msg.texts, title_key ) if results is None: # The model broke the contract (bad segment count, diff --git a/scripts/llm_translator.py b/scripts/llm_translator.py index 91bcbbd..e2b4a7b 100644 --- a/scripts/llm_translator.py +++ b/scripts/llm_translator.py @@ -83,7 +83,8 @@ Rules: - The text uses extended Markdown (container fences ::: name, {...} attributes, task lists, footnotes and more): all of it is formatting syntax and must be preserved exactly — only the human-readable text is translated. - Newlines are significant: a single newline inside a paragraph renders as an actual line break, so keep the line structure exactly and never join, split or rewrap lines. - Preserve the block structure exactly: same blocks separated by blank lines, same headings (# levels), lists, code fences, images and links; do not merge, split, add, drop or reorder blocks. -- Never translate or alter URLs, image destinations, code, or {...} placeholders. Image alt texts and link texts ARE translated.""" +- Never translate or alter URLs, image destinations, code, or {...} placeholders. Image alt texts and link texts ARE translated. +- Prefer established technical loanwords with English roots over forced localizations — the jargon professionals actually use (in Finnish "frontend" becomes "frontti", not "etupääte").""" def article_prompt(target: str, doc: str) -> str: