pagerite/pagerite/segments.py

"""Segmented translation round trip: prose out, translations back in.

A translator model mangles anything that is not plain prose — sentinels get
renumbered, ``![`` becomes sentence punctuation, stray ``<br>`` tags appear.
So the model is never shown any of it: a fragment (a Markdown chunk or a
node title) is parsed with the project's own markdown-it setup
(``markdown.make_md(verbatim=True)`` — extensions included, so container,
attrs, footnote and tasklist syntax never leaks into text tokens) and split
into **prose segments**: the merged text runs, plus image alt texts and
link/image titles. Only those cross the wire, as a plain list of strings
(Job.texts / Result.texts in translate.py) — accompanied, per segment, by
a CONTEXT (Job.contexts): a segment carved out of a larger block (a link
text, a partial run) carries the block's plain text, so the model sees the
sentence it lives in; whole-block segments are self-contextualizing and
carry "". Title fragments carry the article's opening instead (assigned by
the dispatcher from TransItem.context).

Reassembly is server-side offset splicing, not text the model produced:
each segment's source span was located at dispatch (``split``), and
``join`` swaps in the translations. Markup therefore cannot break — it
never left the server. A returned segment must still be pure prose itself
(the model could inject markup INTO a segment); anything else — count
mismatch, empty segment, markup tokens — rejects the whole result and the
fragment stays pending.

Translations legitimately reorder markup within a sentence... but segments
splice back at fixed positions, so a link or image stays where the original
put it. That is the accepted trade-off for never feeding the model markup
(docs/localization.md).

Locating is best effort: a run that is not a verbatim source substring
(entity-decoded text, backslash escapes) is skipped — it simply stays in
the original language. So is any piece containing "<": "<" is the
prose/markup boundary on the wire — translators cut their output there,
so such pieces could not survive the round trip.
"""

import re

from pagerite.markdown import make_md

#: The segmentation parser: the project's own markdown-it, verbatim flavor
#: (see make_md). Never used for rendering.
_MD = make_md(verbatim=True)

#: Any Unicode letter (digits and underscore are not prose).
_LETTER = re.compile(r"[^\W\d_]")

#: A GFM alert marker ([!NOTE] etc.) at the start of a blockquote's first
#: paragraph: syntax, not prose — stripped from the first segment.
_ALERT = re.compile(r"^\[![A-Za-z]+\][ \t]*")

#: Any {...} span: {placeholders} and attrs that ended up inside prose
#: (inline attrs are consumed by the parser; a lone {dates} is not).
_BRACES = re.compile(r"\{[^{}\n]*\}")


def _runs(children: list) -> list[str]:
    """Prose runs of an inline token's children, in order.

    Text tokens merge across soft breaks into one run; every markup token
    (emphasis, links, code, images, HTML, footnote refs, hard breaks) is a
    run boundary. Link and image *text* is prose; autolink text (the URL
    itself) is not. Image tokens contribute their alt-text children and
    their title attribute.
    """
    runs: list[str] = []
    cur: list[str] = []

    def flush() -> None:
        if cur:
            s = "".join(cur)
            cur.clear()
            if _LETTER.search(s):
                runs.append(s)

    skip = 0  # inside an autolink (its text is the URL — not prose)
    for t in children:
        if skip:
            if t.type == "link_close":
                skip -= 1
            continue
        if t.type == "text":
            cur.append(t.content)
        elif t.type == "softbreak":
            cur.append("\n")
        elif t.type == "link_open" and t.markup == "autolink":
            flush()
            skip = 1
        elif t.type == "image":
            flush()
            if t.children:
                runs.extend(_runs(t.children))
            title = t.attrGet("title")
            if title and _LETTER.search(title):
                runs.append(title)
        else:
            flush()
            if t.children:
                runs.extend(_runs(t.children))
    flush()
    return runs


def _block_text(children: list) -> str:
    """The block's text as a reader sees it: text runs and link texts
    merged (softbreaks as newlines); image alts, autolink URLs, code and
    other markup content excluded. Used as the translation CONTEXT for
    segments carved out of the block (link texts, partial runs): a lone
    word translates differently than the same word inside its sentence."""
    parts: list[str] = []
    skip = 0  # inside an autolink (its text is the URL)
    for t in children:
        if skip:
            if t.type == "link_close":
                skip -= 1
            continue
        if t.type == "text":
            parts.append(t.content)
        elif t.type == "softbreak":
            parts.append("\n")
        elif t.type == "link_open" and t.markup == "autolink":
            skip = 1
        elif t.type == "image":
            continue
        elif t.children:
            parts.append(_block_text(t.children))
    return "".join(parts)


def _locate(source: str, needle: str, cursor: int) -> int:
    """The needle's offset in source at/after cursor, -1 when absent.

    An occurrence preceded by a backslash is an escaped character, not the
    token's source: keep looking (failing that, the run is skipped — it
    stays in the original language).
    """
    pos = source.find(needle, cursor)
    while pos > 0 and source[pos - 1] == "\\":
        pos = source.find(needle, pos + 1)
    return pos


def split(text: str) -> tuple[list[tuple[int, int]], list[str], list[str]]:
    """Split a fragment into (spans, segments, contexts): prose segments to
    translate, their byte offsets in ``text`` for splicing the translations
    back, and per-segment translation context.

    Segments containing {...} spans are carved further — the braces stay
    out of the wire text. A run that cannot be located verbatim in the
    source contributes no segment. A segment's context is its block's plain
    text when the segment was carved OUT of a larger block (a link text, a
    partial run); a segment that IS the whole block (a plain paragraph, a
    heading) is self-contextualizing and gets "".
    """
    spans: list[tuple[int, int]] = []
    segments: list[str] = []
    contexts: list[str] = []
    cursor = 0
    blockquote_fresh = 0  # blockquote depth whose first inline is upcoming

    def emit(run: str, at: int, ctx: str) -> None:
        """Carve {...} spans out of the located run; emit the prose pieces,
        stripped — padding whitespace stays in the template, off the wire.
        Pieces containing "<" are never emitted: translators cut output at
        the first "<" (the prose/markup boundary, scripts/translator.py),
        so such a piece could not survive the round trip — it stays in the
        original language instead."""
        pieces = []
        pos = 0
        for m in _BRACES.finditer(run):
            pieces.append((pos, m.start()))
            pos = m.end()
        pieces.append((pos, len(run)))
        for p0, p1 in pieces:
            raw = run[p0:p1]
            piece = raw.strip()
            if _LETTER.search(piece) and "<" not in piece:
                start = at + p0 + (len(raw) - len(raw.lstrip()))
                spans.append((start, start + len(piece)))
                segments.append(piece)
                contexts.append(ctx)

    tokens = _MD.parse(text)
    for t in tokens:
        if t.type == "blockquote_open":
            blockquote_fresh += 1
        elif t.type == "blockquote_close":
            blockquote_fresh -= 1
        elif t.type == "inline":
            kids = t.children or []
            runs = _runs(kids)
            block = _block_text(kids).strip()
            if blockquote_fresh:
                # An alert marker ([!NOTE]) leading the blockquote's first
                # paragraph is syntax; strip it from the segment. (Only the
                # first inline of the blockquote can carry it — the flag
                # clears on the first inline seen.)
                blockquote_fresh = 0
                if runs:
                    run = _ALERT.sub("", runs[0], count=1)
                    if _LETTER.search(run):
                        runs[0] = run
                    else:
                        runs.pop(0)
            for run in runs:
                ctx = block if block and run.strip() != block else ""
                pos = _locate(text, run, cursor)
                if pos != -1:
                    emit(run, pos, ctx)
                    cursor = pos + len(run)
                elif "\n" in run:
                    # Indented continuation lines etc. break the verbatim
                    # match: locate each line separately instead.
                    for part in run.split("\n"):
                        if not _LETTER.search(part):
                            continue
                        pos = _locate(text, part, cursor)
                        if pos != -1:
                            emit(part, pos, ctx)
                            cursor = pos + len(part)
    return spans, segments, contexts


def pure_prose(text: str) -> bool:
    """True when the text parses as nothing but prose (text and softbreak
    tokens) — the acceptance test for a translated segment: the model may
    not return markup of its own (a `<br>` here would splice live HTML into
    the fragment)."""
    children = _MD.parseInline(text)[0].children or []
    return all(t.type in ("text", "softbreak") for t in children)


def join(original: str, spans: list[tuple[int, int]], texts: list[str]) -> str | None:
    """Splice translated segments back into the original fragment; None on
    any validation failure (count mismatch, empty or non-prose segment) —
    the caller drops the result and the fragment stays pending."""
    if len(texts) != len(spans):
        return None
    out: list[str] = []
    cursor = 0
    for (start, end), translation in zip(spans, texts):
        if not translation.strip() or not pure_prose(translation):
            return None
        out.append(original[cursor:start])
        out.append(translation)
        cursor = end
    out.append(original[cursor:])
    return "".join(out)


def has_prose(text: str) -> bool:
    """True when the fragment yields at least one translatable segment.
    Chunks that are all markup, code, placeholders or reference definitions
    have no business reaching the model: every language renders them from
    the original chunk."""
    return bool(split(text)[1])