Segments, not sentinels: prose-only translation wire protocol
- pagerite/segments.py replaces masking.py: a fragment is parsed with the project's own markdown-it (markdown.make_md(verbatim=True), byte-identical tokens) and split into pure-prose segments with source spans; only the segments plus per-segment context surrounds cross the wire (Job.texts / contexts / Result.texts) and translations splice back by offset — markup can no longer break, it never leaves the server. Count/empty/non-prose results are rejected and skipped for the run. - Localization machinery out of app.py: the translator dispatcher (clients, job pipeline, validation skip-list) moves into translate.Dispatcher; translated-edit recording moves into i18n (add_patch, set_title_translation, clear_translations). app.py keeps only the routes. - Structure editor localized: flag strip switches the language titles are shown/edited in (GET /_api/pages?lang= flags translated rows, originals dimmed); retitling in a translation writes a per-language title fragment via StructureOp.lang — slugs, order and hierarchy stay language-independent. - Localization tab: refresh-all button (DELETE /_api/translations) drops machine translations, keeps user patches and clears the skip-list so the dispatcher re-translates everything. - PageEditor always opens in the primary language; editor socket gets reconnect/doc-mismatch logging. Reference translator: per-segment calls with context prompts, deterministic punctuation matching and the "<" markup-bleed cut.
This commit is contained in:
+120
-16
@@ -56,10 +56,59 @@ SEED_X_TAGS = {
|
||||
}
|
||||
SEED_X_NAMES = {v: k for k, v in SEED_X_TAGS.items()}
|
||||
|
||||
#: The fragments are masked Markdown (pagerite/masking.py: ⟦N⟧ sentinels
|
||||
#: stand in for code, URLs, placeholders...); Seed-X has no system prompt,
|
||||
#: so the instruction goes in-line.
|
||||
NOTE = ", preserving all Markdown formatting and keeping every ⟦N⟧ token exactly unchanged"
|
||||
#: The fragments arrive as prose segments (pagerite/segments.py): plain
|
||||
#: text runs only — no markup, URLs, code or placeholders. The wire
|
||||
#: invariant is that segments are PURE PROSE, and one character marks the
|
||||
#: boundary both ways: "<" never appears in a segment. Sources containing
|
||||
#: it are never dispatched (pagerite/segments.py keeps them in the
|
||||
#: original language); the model's output is cut at the first "<" — one
|
||||
#: rule that covers the whole class of markup bleed (an echoed <lang> tag,
|
||||
#: a "<br>", ...) instead of a pattern per artifact. (Generation-level
|
||||
#: stop strings can't do this job: the model's <s> framing token would
|
||||
#: trip a "<" stop at the first token; skip_special_tokens strips the
|
||||
#: framing at decode.)
|
||||
#:
|
||||
#: Two kinds cross the wire (Job.kind), each with its own prompt template:
|
||||
#: titles get told they ARE titles (a lone word otherwise invites
|
||||
#: context-free readings — "About" as "approximately"). Any segment may
|
||||
#: carry its surround in Job.contexts (a title: the article's opening; a
|
||||
#: carved-out segment like a link text: its block's plain text) and is then
|
||||
#: translated together with that surround (seed_x_chunk). No punctuation
|
||||
#: clause, on purpose: Seed-X handles trailing-punctuation instructions by
|
||||
#: slipping into its [COT] reasoning mode (observed for Chinese:
|
||||
#: minutes-long generations, reasoning text in the output) —
|
||||
#: match_punctuation handles stray punctuation deterministically instead.
|
||||
PROMPTS = {
|
||||
"chunk": "Translate the following {source_lang} text into {target_lang}:\n{text} <{tag}>",
|
||||
"title": "Translate the following {source_lang} title into {target_lang}:\n{text} <{tag}>",
|
||||
# Title with the article's opening as context (Job.contexts): the model
|
||||
# translates both; generation stops at the blank line separating them,
|
||||
# and the segment's own part of the output is the translation. No
|
||||
# separator in the output (the model merged them) → seed_x_chunk falls
|
||||
# back to the plain kind template.
|
||||
"title+context": "Translate the following {source_lang} title and the beginning of its article "
|
||||
"into {target_lang}:\n{text}\n\n{context} <{tag}>",
|
||||
# A segment carved out of a larger block (link text, partial run) with
|
||||
# its sentence as context — same mechanics as title+context.
|
||||
"chunk+context": "Translate the following {source_lang} text into {target_lang}:\n"
|
||||
"{text}\n\n{context} <{tag}>",
|
||||
}
|
||||
TERMINAL_PUNCT = ".,!?:;…。，！？；：、"
|
||||
|
||||
|
||||
def match_punctuation(source: str, translated: str) -> str:
|
||||
"""Drop terminal punctuation the model added.
|
||||
|
||||
When the source segment ends without terminal punctuation, the
|
||||
translation must not gain any either. A leading Spanish ¡/¿ only pairs
|
||||
with a terminal !/?, so it goes with it.
|
||||
"""
|
||||
if not source or source[-1] in TERMINAL_PUNCT:
|
||||
return translated
|
||||
trimmed = translated.rstrip(TERMINAL_PUNCT)
|
||||
if trimmed and trimmed[0] in "¡¿":
|
||||
trimmed = trimmed[1:].lstrip()
|
||||
return trimmed
|
||||
|
||||
|
||||
# The wire structs below duplicate pagerite/translate.py: this script runs
|
||||
@@ -79,9 +128,15 @@ class Job(msgspec.Struct, tag="job"):
|
||||
|
||||
lang: str
|
||||
key: bytes #: 9-byte chunk hash (base64 in the JSON frame)
|
||||
text: str #: masked (pagerite/masking.py): the ⟦N⟧ tokens must survive verbatim
|
||||
#: The fragment's prose segments: plain text runs only, no markup —
|
||||
#: translate each element independently (pagerite/segments.py).
|
||||
texts: list[str]
|
||||
path: str #: article it came from ("" = front page), no leading slash
|
||||
kind: str #: "chunk" | "title"
|
||||
#: Per segment (parallel to texts; "" = none): the surround to
|
||||
#: translate it in — a link text carries its sentence, a title the
|
||||
#: article's opening. See seed_x_chunk for how they are used.
|
||||
contexts: list[str] = msgspec.field(default_factory=list)
|
||||
|
||||
|
||||
class Result(msgspec.Struct, tag="result"):
|
||||
@@ -90,7 +145,7 @@ class Result(msgspec.Struct, tag="result"):
|
||||
|
||||
lang: str
|
||||
key: bytes
|
||||
text: str
|
||||
texts: list[str] #: the job's segments translated, same order and count
|
||||
|
||||
|
||||
def load_seed_x():
|
||||
@@ -101,29 +156,78 @@ def load_seed_x():
|
||||
return tokenizer, model
|
||||
|
||||
|
||||
def seed_x_chunk(tokenizer, model, text: str, target_lang: str, tag: str, source_lang: str = "English",
|
||||
note: str = ""):
|
||||
"""Translate one segment; returns (translation, output_tokens, generation_seconds)."""
|
||||
def seed_x_chunk(tokenizer, model, text: str, target_lang: str, tag: str,
|
||||
kind: str = "chunk", context: str = "", source_lang: str = "English"):
|
||||
"""Translate one segment; returns (translation, output_tokens, generation_seconds).
|
||||
|
||||
With context, the segment is translated together with its surround (a
|
||||
link text with its sentence, a title with the article's opening), and
|
||||
the segment's own part of the output is the translation: its own line
|
||||
for a single-line source (a single-line segment's translation never
|
||||
contains a line break — generation stops at the blank line separating
|
||||
the two), its own paragraph for a multi-line one (softbreak-merged
|
||||
lines keep single newlines, the separator is the blank line). If the
|
||||
model merged them — no separator, or an empty first part — fall back to
|
||||
translating the segment alone; the wasted tokens are counted either
|
||||
way.
|
||||
"""
|
||||
# No chat template on this model; the trailing language tag is required (trans/ style prompt).
|
||||
prompt = f"Translate the following {source_lang} text into {target_lang}{note}:\n{text} <{tag}>"
|
||||
template = PROMPTS.get(f"{kind}+context" if context else kind, PROMPTS["chunk"])
|
||||
prompt = template.format(source_lang=source_lang, target_lang=target_lang,
|
||||
text=text, tag=tag, context=context)
|
||||
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
|
||||
t0 = time.monotonic()
|
||||
out = model.generate(**inputs, max_new_tokens=max(1024, 2 * inputs.input_ids.shape[1]), do_sample=False)
|
||||
# The only stop string is the context separator. "<" must NOT be one:
|
||||
# stopping works on the raw output, which always starts with the
|
||||
# model's <s> framing token. skip_special_tokens strips <s>/</s> at
|
||||
# decode; the post-decode cut at the first "<" then enforces the wire
|
||||
# invariant (prose only) against markup bleed.
|
||||
kwargs = {"stop_strings": ["\n\n"], "tokenizer": tokenizer} if context else {}
|
||||
out = model.generate(**inputs, max_new_tokens=max(1024, 2 * inputs.input_ids.shape[1]),
|
||||
do_sample=False, **kwargs)
|
||||
dt = time.monotonic() - t0
|
||||
n = out.shape[1] - inputs.input_ids.shape[1]
|
||||
return tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True).strip(), n, dt
|
||||
decoded = tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
|
||||
translated = decoded.partition("<")[0]
|
||||
if not context:
|
||||
return translated.strip(), n, dt
|
||||
if "\n" in text:
|
||||
# Multi-line segment: its translation keeps single newlines; the
|
||||
# blank line is the separator from the context translation.
|
||||
sep = "\n\n" in translated
|
||||
out = translated.split("\n\n", 1)[0] if sep else ""
|
||||
else:
|
||||
out, sep, _ = translated.partition("\n")
|
||||
if not sep:
|
||||
out = ""
|
||||
out = out.strip()
|
||||
if out:
|
||||
return out, n, dt
|
||||
# The model merged segment and context (no separator, or an empty first
|
||||
# part): retry without the context.
|
||||
again, n2, dt2 = seed_x_chunk(tokenizer, model, text, target_lang, tag,
|
||||
kind=kind, source_lang=source_lang)
|
||||
return again, n + n2, dt + dt2
|
||||
|
||||
|
||||
async def do_job(ws, job: Job, tokenizer, model) -> None:
|
||||
"""Translate the job's one fragment and send the result back."""
|
||||
"""Translate the job's segments (one model call each) and send them back."""
|
||||
lang_name = SEED_X_NAMES[job.lang].capitalize()
|
||||
# Deliberately blocking: nothing else needs the loop while the job is
|
||||
# being answered, and the reconnect loop recovers a dropped connection
|
||||
# (the in-flight item is simply re-offered).
|
||||
text, tokens, dt = seed_x_chunk(tokenizer, model, job.text, lang_name, job.lang, note=NOTE)
|
||||
print(f"[{job.lang} {job.kind} {job.path or '/'}: "
|
||||
texts = []
|
||||
tokens = dt = 0
|
||||
for i, text in enumerate(job.texts):
|
||||
ctx = job.contexts[i] if i < len(job.contexts) else ""
|
||||
translated, n, t = seed_x_chunk(tokenizer, model, text, lang_name, job.lang,
|
||||
kind=job.kind, context=ctx)
|
||||
texts.append(match_punctuation(text, translated))
|
||||
tokens += n
|
||||
dt += t
|
||||
print(f"[{job.lang} {job.kind} {job.path or '/'}: {len(texts)} segments, "
|
||||
f"{tokens} tokens in {dt:.1f}s = {tokens / dt:.1f} tok/s]", file=sys.stderr)
|
||||
await ws.send(msgspec.json.encode(Result(lang=job.lang, key=job.key, text=text)).decode())
|
||||
await ws.send(msgspec.json.encode(Result(lang=job.lang, key=job.key, texts=texts)).decode())
|
||||
|
||||
|
||||
async def serve(url: str, tokenizer, model) -> None:
|
||||
|
||||
Reference in New Issue
Block a user