From 43793ef9a4ae5b187be04152691d889934e3c075 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Thu, 3 Sep 2026 01:46:56 +0000 Subject: [PATCH] Links translated in sentence context, re-linked by text weight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - segments.py: a block of plain text with inline links now stays ONE wire segment (Mark/Span) — the link label crosses in sentence context, so translations come back grammatically coherent instead of incompatible snippets. join() re-inserts the link markdown into the translated block at weight-mapped positions (word units; CJK ideographs and kana runs count one unit each), degrading to the source label on empty slices and still rejecting markup injection. No sentinels on the wire: boundaries are found by text processing alone. - translate.py: the Dispatcher tracks Span (offsets + link marks) for the in-flight job. --- AGENTS.md | 2 +- docs/localization.md | 32 +++-- pagerite/segments.py | 294 +++++++++++++++++++++++++++++++++++++----- pagerite/translate.py | 9 +- 4 files changed, 293 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e0b41ed..2aab7d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). - `i18n.py` — language selection, translation assembly (chunks + patches) and translated-edit recording (user patches, per-language title overrides, refresh). - `translate.py` — translator service protocol (msgspec structs), the connected-client `Dispatcher` (job pipeline, result validation) and pending/store core for the `/_translate/{key}` WebSocket (docs/localization.md); app.py only registers the route. - - `segments.py` — the translation round trip: fragments split into pure-prose wire segments (via markdown.make_md's verbatim parser) and translations spliced back by source offset (docs/localization.md). + - `segments.py` — the translation round trip: fragments split into pure-prose wire segments (via markdown.make_md's verbatim parser; link-carrying blocks stay whole, link texts inline) and translations spliced back by source offset, link markdown re-inserted at weight-mapped positions (docs/localization.md). - `migrations.py` — kanta migrations (`migrate_vN`); ALL schema/storage upgrades live here (raw state dict before struct decoding), never in the app lifespan: v1 moves legacy in-db file blobs to the on-disk store and rebuilds the legacy flat `pages` as the menu tree, v2 rewrites `/_f/{hash}.ext` image links to the extension-less form, backfills AVIF/WebP/JPEG derivatives on disk and drops the obsolete `version` field. - `markdown.py` — markdown-it-py renderer. - `views.py` — shared page layout and rendering; theme/user-font resolution across `THEME_DIRS` / `FONT_DIRS` (cwd, site, platform data roots, then built-in `pagerite/themes/`, see `docs/themes-and-assets.md`). diff --git a/docs/localization.md b/docs/localization.md index ca6191a..fa83c56 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -339,8 +339,10 @@ fragment is parsed with the project's own markdown-it setup (`markdown.make_md(verbatim=True)` — all extensions, but no typographer or tasklist label wrapping, so token text stays byte-identical to the source) and split into the runs a model may touch: paragraph/heading/table-cell text -(merged across soft line breaks), link text, image alt texts and captions, -footnote bodies. Everything else never leaves the server: code spans and +(merged across soft line breaks), image alt texts and captions, footnote +bodies. A block of plain text and inline **links stays whole** — link texts +cross inline, in sentence context (see below). Everything else never leaves +the server: code spans and fences, URLs and autolinks, link/image *destinations*, `{...}` spans (placeholders like `{dates}` as well as attrs), reference and footnote labels, container fences, GFM alert markers (`[!NOTE]`), raw HTML — and all @@ -366,11 +368,24 @@ near-deterministic, so an immediate retry would re-fail; the fragment stays pending and gets another chance on restart or `DELETE /_api/translations`). `Data.trans` therefore only ever holds clean translated Markdown. -The trade-off: segments splice back at fixed positions, so a translation -cannot move a link or image within a sentence — word order around inline -markup follows the original. That is the price for never feeding the model -markup (an earlier sentinel-masking design let the model see and mangle -exactly that punctuation: Seed-X turned `![` into `¡¡…!!`). +Link-carrying blocks are the one place a segment is not spliced verbatim: +a label translated apart from its sentence comes back grammatically +incompatible with it (case government, particles, word order), so the +block crosses whole and the server re-inserts the link markdown into the +translated block. The boundaries are found by **text processing alone** — +markers on the wire are hopeless (an earlier sentinel-masking design let +the model see and mangle exactly that punctuation: Seed-X renumbered the +tokens and turned `![` into `¡¡…!!`). Each link's weight ratio in the +source block (word units before its text boundaries over the block total; +CJK ideographs count as one unit each, kana runs as one — no spaces to +count words by) is applied to the translation's units. Placement is +approximate and drift accumulates across several links in one block — the +accepted trade: better a coherent sentence with a slightly shifted link +than separately translated snippets that don't fit together. A boundary +that maps to an empty slice degrades to the source link text rather than +emitting a broken `[](url)`. Blocks mixing in any other inline markup +(emphasis, code spans, images) don't qualify and still split into runs at +those boundaries. Punctuation is the translator's own job: Seed-X tends to "finish" short labels (titles, nav items) with a comma or period the source never had. @@ -395,7 +410,8 @@ framing token, which would trip a `<` stop immediately.) Short fragments get more than a bare prompt: each segment may carry its surround in `Job.contexts` — a title carries the article's opening prose (its own block is just the title word), a segment carved out of a larger -block (a link text, a partial run) carries the block's plain text, and a +block (a partial run; a link text whose block didn't qualify for the +whole-block treatment) carries the block's plain text, and a whole-block segment (a plain paragraph) is self-contextualizing and carries "". The reference client translates segment and surround together, stops generation at the blank line separating them, and keeps the segment's own diff --git a/pagerite/segments.py b/pagerite/segments.py index 5c5b22a..00dbfd7 100644 --- a/pagerite/segments.py +++ b/pagerite/segments.py @@ -23,10 +23,18 @@ never left the server. A returned segment must still be pure prose itself mismatch, empty segment, markup tokens — rejects the whole result and the fragment stays pending. -Translations legitimately reorder markup within a sentence... but segments -splice back at fixed positions, so a link or image stays where the original -put it. That is the accepted trade-off for never feeding the model markup -(docs/localization.md). +A block of plain text and prose links crosses as ONE segment — link texts +inline, in sentence context — because a label translated apart from its +sentence comes back grammatically incompatible with it (case government, +particles, word order). ``join`` re-inserts the link markdown into the +translated block at weight-mapped positions (``_place_marks``): no markers +on the wire (sentinels never survived the model — they got renumbered and +mangled), the boundaries are found by text processing alone — each link's +word/CJK-char weight ratio in the source applied to the translation's +units. Placement is approximate and CJK-safe: better a coherent sentence +with a slightly shifted link than separately translated snippets that +don't fit together. Blocks with any other inline markup (emphasis, code, +images) still split into runs at those boundaries. Locating is best effort: a run that is not a verbatim source substring (entity-decoded text, backslash escapes) is skipped — it simply stays in @@ -36,6 +44,7 @@ so such pieces could not survive the round trip. """ import re +from typing import NamedTuple from pagerite.markdown import make_md @@ -54,6 +63,59 @@ _ALERT = re.compile(r"^\[![A-Za-z]+\][ \t]*") #: (inline attrs are consumed by the parser; a lone {dates} is not). _BRACES = re.compile(r"\{[^{}\n]*\}") +#: A link's tail after its text: "](dest)", "](dest \"title\")", "][ref]", +#: "[]" or a bare "]" (shortcut reference); the destination may nest one +#: level of parens. Best effort — a mis-scan fails the span-reconstruction +#: check in _linked_block and the block falls back to per-run segments. +_LINK_TAIL = re.compile(r"\](?:\((?:\\.|[^()\\]|\([^()]*\))*\)|\[(?:\\.|[^\]])*\])?") + +#: Weight units for mapping link boundaries from source to translation: +#: a word counts 1 and so does every single CJK ideograph (kana runs count +#: as one) — CJK has no spaces to count words by. Punctuation and +#: whitespace count nothing, so mapped boundaries always land on unit +#: starts. +_UNIT = re.compile( + r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]" # CJK ideographs: one unit each + r"|[\u3040-\u309f\u30a0-\u30ff]+" # kana runs: one unit each + r"|\w+" # anything else word-like (Latin, Cyrillic, Hangul, digits) +) + + +class Mark(NamedTuple): + """One inline link inside a whole-block segment: the source weight + (unit count, see _UNIT) at the link text's start and end for mapping + the boundaries into the translation, the exact source syntax around + the text ("[" / "](url)" etc.) and the source text itself, used as the + fallback when the mapped slice comes out empty (better an untranslated + label than a broken "[](url)").""" + + w_start: int + w_end: int + pre: str + post: str + inner: str + + +class Span(NamedTuple): + """A segment's source span in the fragment: offsets for splicing the + translation back, the segment's source weight and the links to + re-insert into its translation (empty = a plain prose segment).""" + + start: int + end: int + weight: int + marks: list[Mark] + + +def _weight(text: str) -> int: + """The text's weight in translation-mapping units (see _UNIT).""" + return len(_UNIT.findall(text)) + + +def _unit_bounds(text: str) -> list[int]: + """Unit-start offsets of the text, plus its end as the last bound.""" + return [m.start() for m in _UNIT.finditer(text)] + [len(text)] + def _runs(children: list) -> list[str]: """Prose runs of an inline token's children, in order. @@ -141,19 +203,141 @@ def _locate(source: str, needle: str, cursor: int) -> int: return pos -def split(text: str) -> tuple[list[tuple[int, int]], list[str], list[str]]: +def _linked_block( + source: str, kids: list, cursor: int, strip_alert: bool +) -> tuple[Span, str] | None: + """A whole-block segment for an inline of plain text and prose links: + (Span, wire text) with the links as marks, or None when the block has + any other shape — the caller then falls back to per-run segments. + + The block crosses the wire as one prose piece, link texts inline, so a + translation that inflects or reorders around a link stays coherent; + join re-inserts the link markdown at weight-mapped positions. The + source span is located piece by piece and verified by reconstruction; + anything not byte-exact (entities, escapes, an odd link tail) bails to + the fallback. + """ + pieces: list[tuple[str, bool]] = [] # (text, is_link); plain pieces alternate with links + buf: list[str] = [] # current plain piece + link: list[str] | None = None # current link's text parts + for tok in kids: + if tok.type == "link_open": + if link is not None or tok.markup == "autolink": + return None + if buf: + pieces.append(("".join(buf), False)) + buf = [] + link = [] + elif tok.type == "link_close": + if link is None: + return None + inner = "".join(link) + if not _LETTER.search(inner): + return None + pieces.append((inner, True)) + link = None + elif tok.type in ("text", "softbreak"): + (link if link is not None else buf).append( + "\n" if tok.type == "softbreak" else tok.content + ) + else: # emphasis, code, images, HTML, footnote refs: run boundaries + return None + if link is not None: + return None # unbalanced (the parser should not do this) + if buf: + pieces.append(("".join(buf), False)) + if not any(is_link for _, is_link in pieces): + return None + if strip_alert and pieces and not pieces[0][1]: + # A GFM alert marker leading the blockquote's first paragraph is + # syntax; strip it from the wire text (it stays out of the span). + first = _ALERT.sub("", pieces[0][0], count=1) + if first.strip(): + pieces[0] = (first, False) + else: + pieces.pop(0) + if not pieces: + return None + raw = "".join(text for text, _ in pieces) + lead = len(raw) - len(raw.lstrip()) + wire = raw.strip() + if not _LETTER.search(wire) or "<" in wire or _BRACES.search(wire): + return None + # Locate each piece verbatim, in order; the source slices between the + # located pieces are then the link syntax, exact by construction. + located: list[tuple[int, int]] = [] + pos = cursor + for text_, _ in pieces: + at = _locate(source, text_, pos) + if at == -1: + return None + located.append((at, at + len(text_))) + pos = at + len(text_) + span_start, span_end = located[0][0], located[-1][1] + marks: list[Mark] = [] + offset = 0 # raw (pre-strip) plain-text offset of the current piece + for i, ((text_, is_link), (s, e)) in enumerate(zip(pieces, located)): + if not is_link: + offset += len(text_) + continue + # The syntax around the text: the gap between pieces goes to the + # link on its left as post (so between two links the whole "](u)[" + # is the first's post); a block-leading link takes the byte in + # front of its text ("["), a block-trailing one the scanned tail. + if i == 0: + if s == 0: + return None + pre, span_start = source[s - 1:s], s - 1 + elif pieces[i - 1][1]: + pre = "" # the previous link's post covers the whole gap + else: + pre = source[located[i - 1][1]:s] + if i + 1 < len(pieces): + post = source[e:located[i + 1][0]] + else: + m = _LINK_TAIL.match(source, e) + if m is None: + return None + post, span_end = m.group(), m.end() + ps = min(max(offset - lead, 0), len(wire)) + pe = min(max(offset + len(text_) - lead, 0), len(wire)) + if pe <= ps: + return None + marks.append(Mark(_weight(wire[:ps]), _weight(wire[:pe]), pre, post, wire[ps:pe])) + offset += len(text_) + # Verify: the marks must reconstruct the source span exactly (the only + # real risk is the guessed tail of a trailing link). + rec: list[str] = [] + mi = 0 + for text_, is_link in pieces: + if is_link: + mark = marks[mi] + mi += 1 + rec += [mark.pre, text_, mark.post] + else: + rec.append(text_) + if source[span_start:span_end] != "".join(rec): + return None + return Span(span_start, span_end, _weight(wire), marks), wire + + +def split(text: str) -> tuple[list[Span], list[str], list[str]]: """Split a fragment into (spans, segments, contexts): prose segments to - translate, their byte offsets in ``text`` for splicing the translations + translate, their source spans in ``text`` for splicing the translations back, and per-segment translation context. - Segments containing {...} spans are carved further — the braces stay - out of the wire text. A run that cannot be located verbatim in the - source contributes no segment. A segment's context is its block's plain - text when the segment was carved OUT of a larger block (a link text, a - partial run); a segment that IS the whole block (a plain paragraph, a - heading) is self-contextualizing and gets "". + A block of plain text and prose links becomes ONE segment (link texts + inline, in context), the links recorded as marks on its Span for + weight-mapped re-insertion in join. Other blocks split into text runs + at markup boundaries; runs containing {...} spans are carved further — + the braces stay out of the wire text. A run that cannot be located + verbatim in the source contributes no segment. A segment's context is + its block's plain text when the segment was carved OUT of a larger + block (a partial run); a segment that IS the whole block (a plain + paragraph, a heading, a linked block) is self-contextualizing and gets + "". """ - spans: list[tuple[int, int]] = [] + spans: list[Span] = [] segments: list[str] = [] contexts: list[str] = [] cursor = 0 @@ -177,7 +361,7 @@ def split(text: str) -> tuple[list[tuple[int, int]], list[str], list[str]]: piece = raw.strip() if _LETTER.search(piece) and "<" not in piece: start = at + p0 + (len(raw) - len(raw.lstrip())) - spans.append((start, start + len(piece))) + spans.append(Span(start, start + len(piece), 0, [])) segments.append(piece) contexts.append(ctx) @@ -189,20 +373,28 @@ def split(text: str) -> tuple[list[tuple[int, int]], list[str], list[str]]: blockquote_fresh -= 1 elif t.type == "inline": kids = t.children or [] + # An alert marker ([!NOTE]) leading a blockquote's first + # paragraph is syntax; both paths strip it. (Only the first + # inline of the blockquote can carry it — the flag clears on + # the first inline seen.) + alert = bool(blockquote_fresh) + blockquote_fresh = 0 + linked = _linked_block(text, kids, cursor, strip_alert=alert) + if linked is not None: + span, wire = linked + spans.append(span) + segments.append(wire) + contexts.append("") + cursor = span.end + continue runs = _runs(kids) block = _block_text(kids).strip() - if blockquote_fresh: - # An alert marker ([!NOTE]) leading the blockquote's first - # paragraph is syntax; strip it from the segment. (Only the - # first inline of the blockquote can carry it — the flag - # clears on the first inline seen.) - blockquote_fresh = 0 - if runs: - run = _ALERT.sub("", runs[0], count=1) - if _LETTER.search(run): - runs[0] = run - else: - runs.pop(0) + if alert and runs: + run = _ALERT.sub("", runs[0], count=1) + if _LETTER.search(run): + runs[0] = run + else: + runs.pop(0) for run in runs: ctx = block if block and run.strip() != block else "" pos = _locate(text, run, cursor) @@ -231,20 +423,60 @@ def pure_prose(text: str) -> bool: return all(t.type in ("text", "softbreak") for t in children) -def join(original: str, spans: list[tuple[int, int]], texts: list[str]) -> str | None: +def _place_marks(translation: str, weight: int, marks: list[Mark]) -> str | None: + """Re-insert a whole-block segment's links into its translation. + + Each mark's source weight ratio (units before the boundary / total) is + applied to the translation's units — a rough bilingual alignment that + needs no markers in the wire text (sentinels never survived the model) + and works for CJK, where exact placement matters less. A boundary + landing empty degrades to the source link text: better an untranslated + label than a broken "[](url)". None when the translation has no units + to map onto (the caller rejects the result). + """ + bounds = _unit_bounds(translation) + total = len(bounds) - 1 + if not total or not weight: + return None + out: list[str] = [] + cur = 0 + for mark in marks: + x1 = bounds[min(round(mark.w_start / weight * total), total)] + x2 = bounds[min(round(mark.w_end / weight * total), total)] + x1 = max(x1, cur) # monotonic: never before the previous mark's end + x2 = max(x2, x1) + # The slice ends at the next unit's start, so the whitespace before + # that unit is inside it — but it belongs BETWEEN the link and the + # following word, not in the link text: strip it from the link and + # leave it for the following slice (cursor stays ahead of it). + raw = translation[x1:x2] + inner = raw.strip() or mark.inner + out += [translation[cur:x1], mark.pre, inner, mark.post] + cur = x2 - (len(raw) - len(raw.rstrip())) + out.append(translation[cur:]) + return "".join(out) + + +def join(original: str, spans: list[Span], texts: list[str]) -> str | None: """Splice translated segments back into the original fragment; None on any validation failure (count mismatch, empty or non-prose segment) — - the caller drops the result and the fragment stays pending.""" + the caller drops the result and the fragment stays pending. Segments + with marks (a block that crossed as one piece) get their links + re-inserted at weight-mapped positions after the prose check.""" if len(texts) != len(spans): return None out: list[str] = [] cursor = 0 - for (start, end), translation in zip(spans, texts): + for span, translation in zip(spans, texts): if not translation.strip() or not pure_prose(translation): return None - out.append(original[cursor:start]) + if span.marks: + translation = _place_marks(translation, span.weight, span.marks) + if translation is None: + return None + out.append(original[cursor:span.start]) out.append(translation) - cursor = end + cursor = span.end out.append(original[cursor:]) return "".join(out) diff --git a/pagerite/translate.py b/pagerite/translate.py index e4707ef..d6a9c4a 100644 --- a/pagerite/translate.py +++ b/pagerite/translate.py @@ -29,7 +29,7 @@ from pagerite import i18n from pagerite.__main__ import DEFAULT_PORT from pagerite.chunks import chunk_key, needs_translation from pagerite.data import Data, Node, sorted_nodes -from pagerite.segments import join, split +from pagerite.segments import Span, join, split logger = logging.getLogger(__name__) @@ -210,8 +210,9 @@ class _Connection: def __init__(self, capable: set[str]) -> None: self.capable = capable self.inflight: tuple[str, bytes] | None = None - #: Source spans of the in-flight job's segments (splice offsets). - self.spans: list[tuple[int, int]] = [] + #: Source spans of the in-flight job's segments (splice offsets + #: and link marks). + self.spans: list[Span] = [] self.original: str = "" # its full source text (for the splicing) @@ -274,7 +275,7 @@ class Dispatcher: continue inflight = {s.inflight for s in self.clients.values() if s.inflight} job = None - spans: list[tuple[int, int]] = [] + spans: list[Span] = [] original = "" for lang in sorted(langs): for item in pending_items(self.data, lang):