From 91a16f56c5cd3349c5c188a3d688345158803209 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Mon, 21 Sep 2026 18:14:48 +0000 Subject: [PATCH] Redesign translation overrides: keyed, structural, hash-anchored Replace the old "patches" format (path:lang composite keys, ordered hunk lists, text-anchored matching) with Data.overrides: path -> lang -> LangEdits, keyed throughout so a save's database diff touches only the edited chunks. The old "patches" key is ignored on decode, discarding legacy data without a migration. - Whole-paragraph additions/deletions are structural: a drop flag on the original chunk hash, and additions in their own dict anchored from the neighboring chunks' before/after (first live referrer wins), so they stay in place across retranslation and one-sided original edits. - Within-paragraph edits (up to a full paragraph rewrite or split) are full-chunk replace patches applied by chunk hash alone: a retranslation is overridden wholesale, so user edits survive AI re-runs; editing the original changes the hash and orphans the patch. The old search-matching staleness gate is gone. - Saving a translation on a page without original chunks is rejected (REST 400 / WS error); emptying the original afterwards renders the translation empty, with the orphaned overrides inert. --- AGENTS.md | 2 +- docs/llm-translation.md | 21 +- docs/localization.md | 170 ++++++++----- docs/migrate.md | 64 ++--- frontend/src/LocalizationEditor.vue | 4 +- frontend/src/PageEditor.vue | 4 +- pagerite/api.py | 39 +-- pagerite/data.py | 48 +++- pagerite/i18n.py | 358 +++++++++++++++++++++------- pagerite/migrations.py | 12 +- pagerite/translate.py | 14 +- 11 files changed, 500 insertions(+), 236 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 65c26ba..f5d6662 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `pages.py` — public content pages: `/`, `/sitemap.xml`, `/robots.txt`, the `/{path:path}` catch-all. - `data.py` — msgspec Structs for the kanta database. - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). - - `i18n.py` — language selection, translation assembly (chunks + patches) and translated-edit recording (user patches, per-language title overrides, refresh). + - `i18n.py` — language selection, translation assembly (chunks + overrides) and translated-edit recording (per-chunk user overrides in `Data.overrides`, per-language title overrides, refresh). - `translate.py` — translator service protocol (msgspec structs), the connected-client `Dispatcher` (job pipeline, result validation) and pending/store core for the `/_translate/{key}` WebSocket (docs/localization.md); api.py only registers the route. - `segments.py` — the translation round trip: fragments split into pure-prose wire segments (via markdown.make_md's verbatim parser; link- and formatting-carrying blocks stay whole, link/formatted texts inline, Markdown stripped) and translations spliced back by source offset, link/formatting markdown re-inserted at weight-mapped positions (docs/localization.md). - `migrations.py` — kanta migrations (`migrate_vN`); ALL schema/storage upgrades live here (raw state dict before struct decoding), never in the app lifespan: v1 moves legacy in-db file blobs to the on-disk store and rebuilds the legacy flat `pages` as the menu tree, v2 rewrites `/_f/{hash}.ext` image links to the extension-less form, backfills AVIF/WebP/JPEG derivatives on disk and drops the obsolete `version` field. diff --git a/docs/llm-translation.md b/docs/llm-translation.md index a432c50..c58184b 100644 --- a/docs/llm-translation.md +++ b/docs/llm-translation.md @@ -38,7 +38,7 @@ technical with code fences and `{dates}`) into fi/es/zh: - **qwen3.8:27b** (dense, 17 GB Q4 — fits VRAM): structure-perfect on all runs — URLs, placeholders, heading/block counts preserved, fenced code byte-identical. es/zh excellent; fi fluent with occasional lexical slips - (covered by the human patch layer). ~30 s per short article, ~2.5 min + (covered by the human override layer). ~30 s per short article, ~2.5 min for 18 KB. **The reference model for article and markdown modes.** - **qwen3:30b-instruct**: 3× faster, good prose, but rewrote comments and docstrings inside code fences despite explicit instructions — fails @@ -78,9 +78,10 @@ into* it: exactly the edited chunks, all other translations keep applying. - Per-chunk machine translations (`Data.trans[hash][lang]`) and the hybrid render with per-chunk fallback to the original. -- User patches (`Data.patches`) — search/replace hunks over the assembled - hybrid, per-hunk independent and best-effort. Patches are orthogonal to - how `Data.trans` entries were produced. +- User overrides (`Data.overrides`) — per-original-chunk edits + (search/replace pairs, drops, anchored additions) applied structurally to + the assembled hybrid, each independent and best-effort. Overrides are + orthogonal to how `Data.trans` entries were produced. - `pending_items`: after a source edit, exactly the changed (lang, hash) pairs are pending — **focused retranslation of edits falls out of the existing bookkeeping**, no whole-article reruns. @@ -108,11 +109,11 @@ Four job modes, in increasing granularity: a body chunk or a title. `Job.texts` carries a single element, the chunk's Markdown; `Job.contexts` carries up to two context strings (previous and next block of the **served hybrid** in the target - language — current machine translation with user patches applied), + language — current machine translation with user overrides applied), "" where none. The client is instructed to output ONLY the translation of the target block; the context is terminology/tone reference. - Using the *patched* hybrid as context propagates human corrections - into fresh machine translations without the LLM ever touching patch + Using the *overridden* hybrid as context propagates human corrections + into fresh machine translations without the LLM ever touching override storage. `Result.texts` carries one element, the translated block. The server validates: exactly one block after re-chunking, anchor constructs (URLs, image destinations, code fence content, `{...}` @@ -214,7 +215,7 @@ The client announces in `Hello`: - `model`: the model string it is actually serving (e.g. `qwen3.8:27b`) - `langs`: from its per-model language table — for the shipped qwen3.8 configuration the site languages as configured server-side - (de, es, fi, pt, zh; Finnish flagged as the weakest, patch-covered) + (de, es, fi, pt, zh; Finnish flagged as the weakest, override-covered) - `modes`: `["markdown", "article", "nav"]` for a structure-proven model, `["markdown"]` for one that is only trusted in scoped mode @@ -226,9 +227,9 @@ by omitting `modes`). The decomposition function doubles as an import path for translations produced outside the pipeline — e.g. an article translated with ChatGPT and pasted back. Today such a paste lands in the translation editor and -is stored as one giant user patch; feeding it through the same +is stored as one giant set of overrides; feeding it through the same decomposition instead writes proper `Data.trans` fragments, so later source edits invalidate and re-translate per chunk rather than letting -the monolithic patch silently go stale hunk by hunk. This import path is +the monolithic override silently go stale chunk by chunk. This import path is also the natural testbed for the decomposition and validation logic before any live LLM client uses it. diff --git a/docs/localization.md b/docs/localization.md index 8f326cd..19ebeb8 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -7,7 +7,7 @@ parameter or the `Accept-Language` header. plumbing. Translations are consumed through a stub interface; the database still holds only the original language. - **Phase 2 (implemented):** gettext-style fragment storage in the - database — machine-translated chunks plus user override patches, assembled + database — machine-translated chunks plus user overrides, assembled at render time. Storage details in `docs/migrate.md`. ## Phase 1: negotiation and URLs @@ -116,26 +116,28 @@ Region tags normalize to their base subtag (`fi-FI` → `fi`). Phase 1 assumed whole-page translated Markdown delivered from outside. The refined model is gettext-style: an article has **one primary version** (its `content`, in its own language) plus, per target language, **machine -fragments** (translated chunks of Markdown) and **user patches** (minimal -editor overrides). Both are stored in the database and assembled into the -served Markdown at render time. +fragments** (translated chunks of Markdown) and **user overrides** (minimal +editor edits, keyed per original chunk). Both are stored in the database and +assembled into the served Markdown at render time. ### The scenario this must handle 1. Article written in English. 2. Machine-translated into Spanish → fragments stored. 3. Editor fixes one Spanish paragraph and changes a link elsewhere to point - at a Spanish resource → user patch hunks stored. + at a Spanish resource → user overrides stored. 4. English article edited → the edited chunk's key changes; its Spanish fragment no longer matches. 5. Page requested before the machine translation refreshes → served as a **hybrid**: old fragments for unchanged chunks, plain English for the - edited chunk. User patches are attempted against this hybrid, best effort, - each hunk independently: the text fix is stale (its search text no longer - exists) and silently skipped; the link change still applies even though - the link sits in the now-English paragraph. -6. Machine translation refreshes → full Spanish again, with both patch hunks - applying. + edited chunk. User overrides key off chunk hashes, so an override whose + chunk was the edited one is orphaned with the old hash and silently + stops applying; overrides for untouched chunks apply as before, even + over the hybrid. +6. Machine translation refreshes → full Spanish again, with the surviving + overrides applying. An override whose original paragraph was edited + stays orphaned — the edit was about that content — and needs re-doing + when still wanted. ### Chunks @@ -163,41 +165,89 @@ Consequences: - No explicit "source version" bookkeeping is needed — staleness falls out of the keys. -### User patches +### User overrides Editors always edit **full Markdown** in the existing editor UX — never fragments. When editing a translated view (`?lang=es`), the editor is loaded -with the *current hybrid Markdown*; on save, the server computes a minimal -diff against that hybrid and stores it as a patch: +with the *current hybrid Markdown*; on save, the server diffs it against +that hybrid and records the changes as **user overrides**. Storage is keyed +throughout — no lists, no composite keys, no stored ordering: ```python -class Patch(msgspec.Struct, omit_defaults=True): - """One editing session's overrides, applied independently per hunk.""" +class ChunkEdit(msgspec.Struct, omit_defaults=True): + """One original chunk's override in one language.""" - hunks: list[tuple[str, str]] = [] # (search, replace) on hybrid Markdown + replace: str = "" # the user's full text for the chunk + drop: bool = False # the chunk is deleted in this language + before: str = "" # addition ids (LangEdits.adds) inserted + after: str = "" # before/after this chunk + + +class LangEdits(msgspec.Struct, omit_defaults=True): + """All overrides of one article in one language.""" + + chunks: dict[bytes, ChunkEdit] = {} # ORIGINAL chunk hash -> override + adds: dict[str, str] = {} # addition id -> Markdown + + +Data.overrides: dict[str, dict[str, LangEdits]] # path -> lang -> edits ``` -Hunks are produced from `difflib.SequenceMatcher` on the hybrid vs. the -edited text at block granularity: each `replace`/`delete`/`insert` opcode -becomes one `(search, replace)` pair, with the preceding block's tail as -left context for `insert` (pure inserts have empty search context otherwise). -A search text that occurs more than once in the page would hit the first -occurrence at apply time, so ambiguous hunks grow block context (preceding -block first) until unique or the page edge. -Application is dead simple: +Everything keys off the **original chunk hashes**, which already carry the +article's order (`Node.chunks`) — application walks that order, so nothing +about sequence is stored. kanta's change diffs register per key, so a save +touches only the entries for the chunks actually edited (a list would be +rewritten whole every time). -```python -def apply_patch(hybrid: str, patch: Patch) -> str: - for search, replace in patch.hunks: - if search and search in hybrid: - hybrid = hybrid.replace(search, replace, 1) - # missing search text = stale hunk -> silently skipped - return hybrid -``` +The diff runs over the `chunk_markdown` block split +(`difflib.SequenceMatcher`, autojunk off: deterministic, pages are small) +and classifies each opcode per original chunk (`record_override` in +`pagerite/i18n.py`): -Per-hunk independence is the robustness property from the scenario: a stale -text fix does not block a still-valid link change. Patches are stored as an -ordered list and applied in order. +- **Within-paragraph edits** — any `replace`, up to a full rewrite of the + paragraph's text — become the chunk's **`replace`** patch: the user's + text replaces the chunk's served text wholesale, applied by chunk hash + alone. A retranslation of the chunk is overridden wholesale too — the + user's edit stays in effect across AI re-runs; editing the *original* + changes the hash and orphans the patch, so the freshly translated + paragraph reappears (the edit was about that content). A re-edit of the + same chunk **composes** into the patch — repeat edits never need + ordering either. Keyed application also kills the old ambiguity problem: + the patch applies to *its* chunk, never to an identical paragraph + elsewhere by accident. +- **Whole-paragraph deletions** become **`drop`** on the chunk. + Hash-anchored, the deletion survives retranslation untouched (a + text-anchored delete would stop matching and the paragraph would + resurrect); when the *original* paragraph is edited its hash changes and + the freshly translated paragraph reappears — the delete was about that + content, not that position. +- **Whole-paragraph insertions** become **additions** in `adds` under + their own ids, referenced from the neighboring chunks' `before`/`after` + — both, when both exist, and the first live referrer wins at apply time, + so an original edit on one side leaves the other anchor. Since content + hashes don't change under retranslation, the inserted paragraph stays in + place across a refresh. Inserts next to existing addition text splice + into that addition (its text is stable, user-written), as do edits and + deletions of added paragraphs — no original hash is ever needed for + translation-only content. + +A save often mixes several edits. `SequenceMatcher` lumps adjacent changes +into one `replace` opcode, so regions that *removed* blocks are refined +(`_refine_replace`): blocks pair greedily by similarity (ratio ≥ 0.5) into +text edits, leaving unpaired source blocks as deletions — a sentence fix +in the paragraph above a deleted paragraph no longer drags the deletion +into the same patch. The split-paragraph grey case (one paragraph +becomes two) deliberately stays a single `replace` patch holding both +paragraphs: it applies whole across retranslations, rather than +half-applying, and telling a split apart from an edit-plus-insert is +fuzzy anyway. + +Every classification is best effort: a diff position whose base text no +longer matches what the hybrid serves there (the original or the machine +translation moved under an open editor) is skipped rather than recorded +against the wrong chunk. Overrides for hashes the article no longer +contains are harmless orphans (they never apply) and can be +garbage-collected lazily, like orphaned chunks. ### Storage @@ -216,8 +266,9 @@ Full storage design and the `migrate_v3` restructuring live in of the global `ORIGINAL_LANGUAGE` constant. - **Known weakness:** changing a page's (or subtree's) `language` after translations exist mis-keys everything — translations are keyed by - *source* chunks, so old entries silently stop matching and user patches - (searching for old-hybrid text) mostly go stale. That is acceptable: + *source* chunks, so old entries silently stop matching and user + overrides (anchored to the old chunks' hashes) are orphaned. + That is acceptable: the orphaned data is harmless and translations regenerate. We do not migrate translations across a language change. - Article paths are stored and keyed **without leading slashes** @@ -229,27 +280,23 @@ Full storage design and the `migrate_v3` restructuring live in def get_translation(data, path, lang) -> Translation | None: if lang not in node.langs: return None - hybrid = "\n\n".join( - chunks[h] if h in node.no_trans else trans.get(h, {}).get(lang, chunks[h]) - for h in node.chunks - ) - for patch in data.patches.get(f"{path}:{lang}", []): - hybrid = apply_patch(hybrid, patch) - # Deleting an extra (translation-only) paragraph leaves its surrounding - # blank lines behind; re-chunking normalizes them away. - hybrid = join_chunks(chunk_markdown(hybrid)) + hybrid = hybrid_markdown(data, node, path, lang) # i18n.py: walk + # node.chunks; per chunk chunks[h] if h in node.no_trans else + # trans.get(h, {}).get(lang, chunks[h]), with the chunk's override + # applied structurally: its before-addition, the chunk itself (dropped, + # or replaced wholesale by the edit's `replace`), its after-addition. return Translation(markdown=hybrid, titles=title_map(data, lang)) ``` - Availability is an article-level index: `node.langs: dict[lang, True]`, - maintained by the translation writers (translator job, patch saves) in the - same transaction as their data writes — rendering and language selection + maintained by the translation writers (translator job, override saves) in + the same transaction as their data writes — rendering and language selection never probe the `trans` store chunk by chunk. A stale key is benign (the "translation" just renders as the original). - `titles` for nav/sidebar/cards: each node's translated title is `trans.get(hash(node.title), {}).get(lang)` with per-node fallback — one dict lookup per nav item at render time. -- Cache invalidation: writes to `chunks` / `trans` / `patches` (translator, +- Cache invalidation: writes to `chunks` / `trans` / `overrides` (translator, editor saves) call `_invalidate_pages()`, same as content writes. ### Editor flow @@ -281,17 +328,22 @@ preferences. metadata (`lang`, `primary_lang`, `langs`, `translate_langs`). - The editor keeps a **shadow copy** of the Markdown it opened. WS `save` with `lang` sends it as `base`; the server diffs `base` → submitted text - (`make_patch`) and appends a `Patch`. Diffing against the shadow (rather - than the current hybrid) keeps hunks correct when the original or the - machine translation moved under an open editor; application against the - then-current hybrid stays best-effort per hunk, as designed. + (`record_override`) and stores per-chunk overrides. Diffing against the + shadow (rather than the current hybrid) keeps the diff correct when the + original or the machine translation moved under an open editor; positions + that no longer match the then-current hybrid are skipped, as designed. - A changed **title** on a translated save becomes a fragment in `Data.trans` keyed by the original title's chunk hash — the same storage as machine title translations. An untouched title field (holding the served translation) is not sent, so saving never freezes a stale machine title into an override. - Saving never deletes; a translation additionally cannot be emptied (that - would render as a blank page in that language). + would render as a blank page in that language), and a translated save on + a page without original content is rejected outright (there is nothing + to anchor a translation to — "the page has no content to translate"). + The converse is fine: if the original is edited empty after the fact, + every override's anchor is gone and the translation simply renders + empty, its overrides inert orphans. - The live preview renders the version being edited, whichever language the page itself was loaded in (the render is just the edited Markdown + title). A translated save keeps that preview in place — re-fetching the @@ -367,7 +419,7 @@ simply stays idle. `DELETE /_api/translations` (the localization tab's "refresh all translations" button) drops every machine translation (`Data.trans`) and rebuilds the availability index (`node.langs`) from the surviving user -patches, so the dispatcher re-translates everything from scratch; the +overrides, so the dispatcher re-translates everything from scratch; the run's validation skip-list is cleared with it, giving rejected fragments another chance. @@ -410,7 +462,7 @@ offerable to clients of another approach. single element, the chunk (a title crosses as plain text, with the article's opening as its context as today). `Job.contexts` carries the previous and next block of the **served hybrid** in the target language - (machine translation with user patches applied, "" where none), so human + (machine translation with user overrides applied, "" where none), so human corrections propagate into fresh translations as terminology/tone reference; contexts are never part of the result. The result must re-chunk to exactly one block with the source's anchor constructs (link @@ -461,8 +513,8 @@ import path — `scripts/import_translation.py PATH LANG FILE.md` (run with the server stopped) decomposes a pasted whole-article translation (e.g. from ChatGPT) into proper `Data.trans` fragments with the same validation, so later source edits invalidate and re-translate per chunk rather than -letting the translation editor's one monolithic patch go stale hunk by -hunk. +letting the translation editor's one monolithic override go stale chunk by +chunk. #### Segmentation diff --git a/docs/migrate.md b/docs/migrate.md index e57e5df..aa5b5cb 100644 --- a/docs/migrate.md +++ b/docs/migrate.md @@ -67,9 +67,12 @@ class Data(msgspec.Struct): #: (nested, not tuple keys: msgspec's JSON serializer rejects them). #: Also used for node titles (hash of the title text). trans: dict[bytes, dict[str, str]] = {} - #: User override patches per article and language: - #: f"{path}:{lang}" -> ordered patches (see localization.md). - patches: dict[str, list[Patch]] = {} + #: User override edits per article and language: + #: path -> lang -> LangEdits (see localization.md) — keyed per original + #: chunk hash throughout, so a save's change diff touches only the + #: edited chunks. Replaced the old list-valued "patches" key (ignored + #: on decode, discarding that data — no migration). + overrides: dict[str, dict[str, LangEdits]] = {} ``` Notes: @@ -88,13 +91,14 @@ Notes: (keyed by chunk hash, so a heavy edit silently drops the flag — acceptable and self-healing). -- **Patch payloads stay inline** in `Patch.hunks` — patches are small by - construction (minimal server-computed diffs). If a pathological case shows - up, hunks can be hash-stored later without schema pain. +- **Override payloads stay inline** in the `LangEdits` struct — overrides + are small by construction (minimal server-computed diffs). If a + pathological case shows up, they can be hash-stored later without schema + pain. ## Language index maintenance (`node.langs`) -`node.langs` is a denormalized index over the `trans`/`patches` stores so +`node.langs` is a denormalized index over the `trans`/`overrides` stores so that article rendering, `select_language`'s availability check, and hreflang alternate links never enumerate chunks. It is written by whoever writes translation data, in the same transaction: @@ -107,11 +111,11 @@ translation data, in the same transaction: writes the `trans[h][lang]` entry, sets `node.langs[lang] = True` on every article that gained one and invalidates the page cache — all in one transaction. -- **Translated-view save:** appending the first patch for `f"{path}:{lang}"` - sets `node.langs[lang] = True` (patches alone make the version exist). -- **Removals:** deleting a patch or GC'ing translations re-derives the key: +- **Translated-view save:** recording the first override for a `(path, lang)` + sets `node.langs[lang] = True` (overrides alone make the version exist). +- **Removals:** deleting overrides or GC'ing translations re-derives the key: keep `lang` if any `trans` entry for the article's current chunks/title or - any patch remains, otherwise drop it. Stale `langs` keys are benign (an + any override remains, otherwise drop it. Stale `langs` keys are benign (an advertised language that renders as the original), so removal can lag. ## Render / save pipeline (summary) @@ -119,8 +123,10 @@ translation data, in the same transaction: - **Render:** `text = "\n\n".join(chunks[h] for h in node.chunks)` for the original; for language `L` (only ever attempted when `L in node.langs`), per chunk `trans.get(h, {}).get(L)` unless missing or `h in node.no_trans`, - falling back to `chunks[h]`; then apply `patches.get(f"{path}:{L}", [])` - in order (per-hunk, best effort); then `markdown.render` as today. All of + falling back to `chunks[h]`; then apply `overrides[path][L]` structurally + in the article's own chunk order (drops, search/replace pairs, anchored + additions — see docs/localization.md); then + `markdown.render` as today. All of this assembles the `Translation` the phase-1 plumbing already consumes. - **Availability:** `node.langs` is the availability index; `?lang=` handling uses exactly this set. (hreflang alternates are site-wide from @@ -128,9 +134,9 @@ translation data, in the same transaction: - **Save (primary language):** server re-chunks the submitted Markdown, inserts new hashes into `Data.chunks`, replaces `node.chunks`. Unchanged chunks keep their hashes — only genuinely new text lands in the diff. -- **Save (translated view):** diff against the served hybrid, append a - `Patch` under `patches[f"{path}:{lang}"]`; `node.chunks` untouched. -- **Invalidate:** any write to `chunks` / `trans` / `patches` calls +- **Save (translated view):** diff against the served hybrid, record + per-chunk overrides under `overrides[path][lang]`; `node.chunks` untouched. +- **Invalidate:** any write to `chunks` / `trans` / `overrides` calls `_invalidate_pages()`. ## migrate_v3 steps @@ -138,10 +144,8 @@ translation data, in the same transaction: 1. Walk `menu`; for every node with a string `content`: `chunks = chunk_markdown(content)`; write each into the new `chunks` store; replace the field with the hash list (`None` stays `None`). -2. Initialize empty `chunks` / `trans` / `patches` stores. -3. Normalize stored paths: strip leading slashes anywhere paths are keys or - values. -4. `language`, `no_trans` and `langs` need nothing — struct defaults cover +2. Initialize empty `chunks` / `trans` stores. +3. `language`, `no_trans` and `langs` need nothing — struct defaults cover them (`langs` starts empty; the translator job fills it as translations land). @@ -161,19 +165,19 @@ Chunking must be deterministic and shared with render/save, so - `Translation.titles` stayed keyed by node path (phase-1 shape, views untouched): `get_translation` builds it by walking the menu with the same per-title `trans.get(chunk_key(node.title), {}).get(lang)` lookups. -- Insert hunks anchor on the whole preceding block (not just its tail) — - a stronger, simpler search context. -- `make_patch` diffs with `SequenceMatcher(autojunk=False)` so patches are - deterministic (popular lines like blank separators never become junk). -- Step 3's path normalization is a no-op in practice: the only path-keyed - store (`patches`) starts empty at v3; analytics paths live outside the - kantadb. The code still strips leading slashes defensively. +- User overrides (`record_override`) diff with `SequenceMatcher(autojunk=False)` + so overrides are deterministic (popular lines like blank separators never + become junk). +- The old list-valued `patches` store was later replaced by the keyed + `overrides` store above; the rename itself discarded the old data (msgspec + ignores the unknown key on decode), no migration. ## Garbage collection (later, manual or idle-time) Orphaned entries accumulate: chunks no longer referenced by any -`node.chunks`/`node.title`, translations whose chunk hash is orphaned, patch -hunks that never match. All are harmless (never read). A GC pass is a single +`node.chunks`/`node.title`, translations whose chunk hash is orphaned, +overrides whose chunk hash is gone from the article (or whose `search` +never matches). All are harmless (never read). A GC pass is a single tree walk collecting live hashes, then deleting the rest from `chunks` and -`trans`; patches whose every hunk is stale get pruned. Not part of +`trans`; override entries for dead hashes get pruned. Not part of migrate_v3. diff --git a/frontend/src/LocalizationEditor.vue b/frontend/src/LocalizationEditor.vue index 410b0d1..da3b212 100644 --- a/frontend/src/LocalizationEditor.vue +++ b/frontend/src/LocalizationEditor.vue @@ -7,7 +7,7 @@ // Flag clicks toggle and save immediately; the settings round-trip // re-reads the payload, so this tab only ever changes translate_langs. The // settings write's invalidation hook kicks the translation dispatcher. The -// refresh button drops all machine translations (user patches are kept), +// refresh button drops all machine translations (user overrides are kept), // making the dispatcher re-translate everything. Translator keys are // managed inline (➕ add, name edit, ✕ delete); new keys are generated // here in the server's format and everything rides the settings @@ -100,7 +100,7 @@ async function toggle(code) { // Delete all machine translations server-side; the dispatcher re-fills // them (a connected translator starts getting jobs right away). User -// patches survive — they are edits, not machine output. +// overrides survive — they are edits, not machine output. const refreshing = ref(false) async function refresh() { if (refreshing.value) return diff --git a/frontend/src/PageEditor.vue b/frontend/src/PageEditor.vue index 1c4a2a3..1a78ec6 100644 --- a/frontend/src/PageEditor.vue +++ b/frontend/src/PageEditor.vue @@ -20,7 +20,7 @@ // edited as its effective (hybrid) Markdown; the hybrid the session // started from is kept as a shadow copy (shadowBase) and sent along at // save time, so the server diffs the user's changes only and stores them -// as a patch — edits to a translation never touch the original, while +// as per-chunk overrides — edits to a translation never touch the original, while // edits to the primary language re-chunk the original (and thereby // invalidate the affected translation fragments). The live preview always // renders the version being edited, whichever language the page itself @@ -208,7 +208,7 @@ function save() { if (lang.value) { msg.lang = lang.value // The shadow copy this session started from: the server diffs base → - // markdown and stores only the user's changes as a patch. + // markdown and stores only the user's changes as overrides. msg.base = shadowBase // An untouched title field is not sent: it holds the served // translation, which a save must not freeze into an override fragment. diff --git a/pagerite/api.py b/pagerite/api.py index 022342f..5ea110f 100644 --- a/pagerite/api.py +++ b/pagerite/api.py @@ -112,10 +112,9 @@ async def save_page( With a ``?lang=`` query (a translation, not the primary language) the save is a translated-view edit (docs/localization.md): the markdown is - diffed against the currently served hybrid and the minimal diff is - appended as a Patch under ``patches[f"{path}:{lang}"]`` — node.chunks - and the original-language fields (title, published, banner) stay - untouched. + diffed against the currently served hybrid and recorded as user + overrides under ``overrides[path][lang]`` — node.chunks and the + original-language fields (title, published, banner) stay untouched. """ path = path.strip("/") _check_reserved(path) @@ -125,11 +124,13 @@ async def save_page( node = chain[-1] if chain else None if node is None or node.chunks is None: raise HTTPException(404, "no such page") + if not node.chunks: + raise HTTPException(400, "the page has no content to translate") with kanta.transaction( f"page:{lang}", user=request.headers.get("remote-user"), extra=path ): - # Patches alone make the translated version exist. - if i18n.add_patch(data, node, path, lang, page.markdown): + # Overrides alone make the translated version exist. + if i18n.record_override(data, node, path, lang, page.markdown): _invalidate_pages() return with kanta.transaction("page", user=request.headers.get("remote-user"), extra=path): @@ -331,9 +332,9 @@ async def delete_translations(request: Request) -> None: """Drop all machine translations (Data.trans) so the dispatcher re-translates everything from scratch (a translate:reset action: the invalidation hook re-offers every fragment to connected - translators). User patches are kept; the availability index - (node.langs) is rebuilt from them — patches alone still make a language - exist on a page.""" + translators). User overrides are kept; the availability index + (node.langs) is rebuilt from them — overrides alone still make a + language exist on a page.""" with kanta.transaction("translate:reset", user=request.headers.get("remote-user")): i18n.clear_translations(data) _invalidate_pages() @@ -422,7 +423,7 @@ async def editor_ws(ws: WebSocket) -> None: effective hybrid Markdown and title for that language plus the language metadata the picker's UI needs; save diffs the submitted Markdown against "base" (the editor's shadow copy of the hybrid it started from - — absent: the current hybrid) and stores it as a user Patch, and a + — absent: the current hybrid) and records it as user overrides, and a changed title becomes a fragment in Data.trans — node.chunks and the other fields stay untouched (docs/localization.md). """ @@ -453,7 +454,7 @@ async def editor_ws(ws: WebSocket) -> None: if lang and node.chunks is not None: # Translation view: the effective (hybrid) # Markdown and title for that language — - # machine fragments + user patches over the + # machine fragments + user overrides over the # original (docs/localization.md editor flow). markdown = i18n.hybrid_markdown(data, node, path, lang) title = i18n.title_map(data, lang).get(path) or title @@ -631,6 +632,16 @@ async def editor_ws(ws: WebSocket) -> None: # original; it cannot create or move pages. await ws.send_json({"type": "error", "detail": "no such page"}) continue + if translated and not old.chunks: + # Nothing to anchor a translation to: the original + # page has no content. + await ws.send_json( + { + "type": "error", + "detail": "the page has no content to translate", + } + ) + continue if translated and "markdown" in msg and not msg["markdown"].strip(): # Saving never deletes; an emptied translation would # render as a blank page in that language. @@ -692,13 +703,13 @@ async def editor_ws(ws: WebSocket) -> None: # node.chunks and the original-language fields # stay untouched: the markdown diff (against the # editor's shadow "base" — the hybrid it started - # from; absent: the current hybrid) is appended - # as a Patch, a changed title becomes a + # from; absent: the current hybrid) is recorded + # as user overrides, a changed title becomes a # per-language title override (i18n). changed = False if "markdown" in msg: base = msg.get("base") - changed = i18n.add_patch( + changed = i18n.record_override( data, node, path, diff --git a/pagerite/data.py b/pagerite/data.py index 2b610bb..bb9f48d 100644 --- a/pagerite/data.py +++ b/pagerite/data.py @@ -15,12 +15,39 @@ import msgspec from pagerite.chunks import join_chunks -class Patch(msgspec.Struct, omit_defaults=True): - """One editing session's overrides on a translated view, applied - independently per hunk (docs/localization.md).""" +class ChunkEdit(msgspec.Struct, omit_defaults=True): + """One original chunk's user override in one language + (docs/localization.md). Fields are independent and applied by chunk + hash alone; an entry for a hash the article no longer contains simply + never applies.""" - #: (search, replace) pairs on the served hybrid Markdown. - hunks: list[tuple[str, str]] = [] + #: Full-chunk replacement text, applied whenever the article still + #: contains the chunk — a retranslation of the chunk is overridden + #: wholesale (editing the original changes the hash, orphaning the + #: patch). May contain blank lines (a paragraph split). A re-edit of + #: the chunk composes into this text. + replace: str = "" + #: The chunk is deleted in this language. Hash-anchored, so the + #: deletion survives retranslation; when the original paragraph itself + #: is edited its hash changes and the fresh translation reappears. + drop: bool = False + #: Addition ids (LangEdits.adds) inserted before/after this chunk. + before: str = "" + after: str = "" + + +class LangEdits(msgspec.Struct, omit_defaults=True): + """All user overrides of one article in one language. Keyed throughout + (no lists), so a save's database diff touches only the edited chunks; + application order comes from the article's own chunk order.""" + + #: Original chunk hash -> override. + chunks: dict[bytes, ChunkEdit] = {} + #: Translation-only additions: id -> Markdown, one per insertion gap, + #: referenced from the neighboring chunks' ``before``/``after`` (both + #: point at the same id; the first live referrer wins at apply time, so + #: an original edit on one side leaves the other anchor). + adds: dict[str, str] = {} class Node(msgspec.Struct, omit_defaults=True): @@ -74,7 +101,9 @@ class Node(msgspec.Struct, omit_defaults=True): #: down the tree (unlike image). large: bool | None = None published: bool = True - children: dict[str, Node] = {} + # Quoted: msgspec 0.21 evaluates the bare self-reference eagerly at + # class creation (Python 3.14 lazy annotations) and NameErrors. + children: dict[str, "Node"] = {} # noqa: UP037 created: datetime = msgspec.field( default_factory=lambda: datetime.now(UTC), ) @@ -132,9 +161,10 @@ class Data(msgspec.Struct): #: serializer does not support). Also used for node titles (hash of #: the title text). trans: dict[bytes, dict[str, str]] = {} - #: User override patches per article and language: - #: f"{path}:{lang}" -> ordered patches (paths without leading slash). - patches: dict[str, list[Patch]] = {} + #: User override edits per article and language: + #: path -> lang -> LangEdits (paths without leading slash). Replaces + #: the old "patches" key (ignored on decode, discarding that data). + overrides: dict[str, dict[str, LangEdits]] = {} def node_markdown(data: Data, node: Node) -> str | None: diff --git a/pagerite/i18n.py b/pagerite/i18n.py index 57fa203..36612ea 100644 --- a/pagerite/i18n.py +++ b/pagerite/i18n.py @@ -5,17 +5,18 @@ language is ``Node.language``, inherited down the hierarchy (front page = site default, ORIGINAL_LANGUAGE as the final fallback). The database holds the original language as content-addressed chunks (``Data.chunks``); per target language there are machine-translated fragments (``Data.trans``) -and user override patches (``Data.patches``), assembled into the served +and user overrides (``Data.overrides``), assembled into the served Markdown at render time, with per-node fallback to the original titles. """ +import secrets from collections.abc import Callable from difflib import SequenceMatcher import msgspec from pagerite.chunks import chunk_key, chunk_markdown, join_chunks -from pagerite.data import Data, Node, Patch, resolve +from pagerite.data import ChunkEdit, Data, LangEdits, Node, resolve #: Final fallback for a page's primary language when neither it nor any #: ancestor (up to the front page) sets one (Node.language, "" = inherit). @@ -101,102 +102,285 @@ def select_language( return original -def apply_patch(hybrid: str, patch: Patch) -> str: - """Apply one patch to the hybrid Markdown, best effort, each hunk - independently: a hunk whose search text no longer exists is stale and - silently skipped (docs/localization.md).""" - for search, replace in patch.hunks: - if search and search in hybrid: - hybrid = hybrid.replace(search, replace, 1) - return hybrid +def hybrid_items(data: Data, node: Node, path: str, lang: str) -> list[tuple[bytes | None, str]]: + """The served hybrid as (anchor, block text) pairs: the anchor is the + ORIGINAL chunk hash behind the block (None for translation-only + addition blocks), in article order. - -def make_patch(base: str, edited: str) -> Patch: - """The minimal diff of ``edited`` against the served ``base`` hybrid as - (search, replace) hunks at block granularity (docs/localization.md). - - Blocks are the chunk_markdown split, so hunks align with translation - units and code fences never straddle a hunk boundary. Pure inserts - anchor on the preceding block (an empty search would never match); - inserts at the very top anchor on the first block. A search text that - occurs more than once in the page would hit the FIRST occurrence at - apply time (apply_patch replaces once) — possibly the wrong instance — - so ambiguous hunks grow block context (preceding block first) until - unique or the page edge, at the cost of going stale when a neighbor - block changes. autojunk is off: the diff must be deterministic, and - pages are small. + User overrides (``Data.overrides``) are structural: walking the + article's own chunk order, each original chunk contributes its + before-addition, the chunk itself (dropped, or its text replaced + wholesale by the edit's ``replace``), and its after-addition. An + override for a hash the article no longer contains never applies; an + addition id referenced from two neighbors is emitted once, at the + first live referrer. """ - a, b = chunk_markdown(base), chunk_markdown(edited) - hunks: list[tuple[str, str]] = [] - for tag, i1, i2, j1, j2 in SequenceMatcher( - None, a, b, autojunk=False - ).get_opcodes(): - if tag == "equal": - continue - core = "\n\n".join(b[j1:j2]) - if tag == "insert": - left, right = (i1 - 1, i1) if i1 else (0, 1 if a else 0) - # Empty base: left == right == 0, the search stays empty and the - # hunk is inert (apply_patch skips empty searches); saving a - # translation of an empty page records nothing applicable. - else: - left, right = i1, i2 - while True: - search = "\n\n".join(a[left:right]) - if not search or base.count(search) <= 1: - break - if left == 0 and right == len(a): - break # whole page and still ambiguous: best effort - if left: - left -= 1 - else: - right += 1 - replace = "\n\n".join([*a[left:i1], *([core] if core else []), *a[i2:right]]) - hunks.append((search, replace)) - return Patch(hunks=hunks) + le = (data.overrides.get(path) or {}).get(lang) + items: list[tuple[bytes | None, str]] = [] + emitted: set[str] = set() + + def emit_add(add_id: str) -> None: + if le and add_id not in emitted and (md := le.adds.get(add_id)): + emitted.add(add_id) + items.extend((None, block) for block in chunk_markdown(md)) + + for h in node.chunks or []: + edit = le.chunks.get(h) if le else None + if edit is not None: + emit_add(edit.before) + if edit is None or not edit.drop: + text = ( + data.chunks.get(h, "") + if h in node.no_trans + else data.trans.get(h, {}).get(lang) or data.chunks.get(h, "") + ) + if edit is not None and edit.replace: + text = edit.replace + items.extend((h, block) for block in chunk_markdown(text)) + if edit is not None: + emit_add(edit.after) + return items def hybrid_markdown(data: Data, node: Node, path: str, lang: str) -> str: """The served Markdown for ``lang``: per chunk the translation from ``Data.trans``, unless missing or marked no-translate (fallback to the - original chunk), then the language's user patches applied in order. + original chunk), with the language's user overrides applied + structurally (hybrid_items). Not gated on ``node.langs`` (get_translation is the gated view): the - editor save path diffs against this even for a language's first patch. + editor save path diffs against this even for a language's first edit. """ - hybrid = join_chunks( - [ + return join_chunks([text for _, text in hybrid_items(data, node, path, lang)]) + + +#: Minimum block similarity for two blocks in a shrunk replace region to +#: pair as a text edit (a per-chunk replace patch) rather than a +#: drop + insertion (_refine_replace). +_PAIR_MIN = 0.5 + + +def _refine_replace( + a: list[str], i1: int, i2: int, b: list[str], j1: int, j2: int +) -> list[tuple[str, int, int, int, int]]: + """Split a ``replace`` opcode that removed blocks (more source than + edited blocks) into single-block sub-opcodes: greedily pair the most + similar source/edited blocks as text edits — a sentence fixed in the + paragraph above a deleted paragraph must not drag the deletion into + the same replace pair — leaving unpaired source blocks as deletions + and any unpaired edited blocks as insertions. + + Only shrunk regions are refined: 1:1 replacements (up to a full + paragraph rewrite) and paragraph splits stay single replace pairs by + design. Regions are a handful of blocks, so the O(n*m) pairing with a + character-level ratio per candidate is cheap, and pages are small, so + the greedy best-first order is deterministic enough. + """ + paired: list[tuple[int, int]] = [] + left_a = list(range(i1, i2)) + left_b = list(range(j1, j2)) + while left_a and left_b: + ratio, ai, bj = max( + (SequenceMatcher(None, a[x], b[y], autojunk=False).ratio(), x, y) + for x in left_a + for y in left_b + ) + if ratio < _PAIR_MIN: + break + paired.append((ai, bj)) + left_a.remove(ai) + left_b.remove(bj) + ops = [] + for ai, bj in paired: + ops.append((ai, bj, ("replace", ai, ai + 1, bj, bj + 1))) + for ai in left_a: + ops.append((ai, j1, ("delete", ai, ai + 1, j1, j1))) + for bj in left_b: + # Anchor an unpaired insertion just after the nearest preceding + # paired source block (the region start when none). + pos = max((ai + 1 for ai, prev in paired if prev < bj), default=i1) + ops.append((pos, bj, ("insert", pos, pos, bj, bj + 1))) + return [op for _, _, op in sorted(ops, key=lambda e: (e[0], e[1]))] + + +def record_override( + data: Data, node: Node, path: str, lang: str, edited: str, base: str | None = None +) -> bool: + """Record a translated-view edit as user overrides (``Data.overrides``): + the block-level diff of ``edited`` against ``base`` (default: the + currently served hybrid), classified per original chunk (docs/ + localization.md): + + - a changed block becomes its chunk's full-text ``replace`` patch — a + re-edit composes into the patch; + - a removed block becomes its chunk's ``drop``; + - new blocks become an addition in ``adds``, anchored from the + neighboring chunks' ``before``/``after`` (inserts next to existing + addition text splice into that addition instead). + + Each save touches only the keys of the chunks actually edited. Every + classification is best effort: a diff position whose base text no + longer matches what the hybrid serves there (the original or the + machine translation moved under an open editor) is skipped rather than + recorded against the wrong chunk. Overrides alone make the translated + version exist, so ``node.langs`` is set. Returns True when anything + was recorded. Pure data ops — the caller wraps in a transaction and + invalidates. + + The callers reject pages without original chunks (there is nothing to + anchor a translation to); should one slip through, the diff finds no + anchors and nothing is recorded. + """ + le = (data.overrides.get(path) or {}).get(lang) + items = hybrid_items(data, node, path, lang) + a = chunk_markdown(base) if base is not None else [text for _, text in items] + b = chunk_markdown(edited) + aligned = len(a) == len(items) + changed = False + + def edits() -> LangEdits: + nonlocal le + if le is None: + le = data.overrides.setdefault(path, {}).setdefault(lang, LangEdits()) + return le + + def anchor_at(i: int) -> bytes | None: + return items[i][0] if aligned else None + + def verified(i: int) -> bool: + """The diff position still holds the text the hybrid serves there + (False when the original or the translation moved under an open + editor — structural ops against a shifted position are skipped).""" + return aligned and a[i] == items[i][1] + + def served(h: bytes) -> str: + return ( data.chunks.get(h, "") if h in node.no_trans else data.trans.get(h, {}).get(lang) or data.chunks.get(h, "") - for h in node.chunks or [] - ] - ) - for patch in data.patches.get(f"{path}:{lang}", []): - hybrid = apply_patch(hybrid, patch) - # A patch deleting an extra (translation-only) paragraph removes its - # text but not one of the surrounding separators, leaving a stray blank - # line behind (apply_patch is a plain string replace). Re-chunk to - # normalize blank lines away — fence/HTML-atomic, and it repairs gaps - # left by patches stored before this normalization. - return join_chunks(chunk_markdown(hybrid)) + ) + def find_add(block: str) -> tuple[str, list[str]] | None: + """(id, blocks) of the addition containing ``block`` (exact block + match — addition text is stable, user-written).""" + if le: + for add_id, md in le.adds.items(): + blocks = chunk_markdown(md) + if block in blocks: + return add_id, blocks + return None -def add_patch( - data: Data, node: Node, path: str, lang: str, edited: str, base: str | None = None -) -> bool: - """Record a translated-view edit as a user Patch: the minimal diff of - ``edited`` against ``base`` (default: the currently served hybrid), - appended to the language's patch list. Patches alone make the - translated version exist, so ``node.langs`` is set. Returns True when - a patch was stored. Pure data ops — the caller wraps in a transaction - and invalidates.""" - patch = make_patch( - base if base is not None else hybrid_markdown(data, node, path, lang), edited - ) - if not patch.hunks: + def do_delete(i: int) -> None: + nonlocal changed + h = anchor_at(i) + if h is not None: + if not verified(i): + return + ce = edits().chunks.setdefault(h, ChunkEdit()) + ce.drop = True + ce.replace = "" + elif found := find_add(a[i]): + add_id, blocks = found + blocks.remove(a[i]) + if blocks: + edits().adds[add_id] = "\n\n".join(blocks) + else: + del edits().adds[add_id] + else: + return + changed = True + + def do_insert(i1: int, new_blocks: list[str]) -> None: + nonlocal changed + left, right = i1 > 0, i1 < len(a) + # Next to existing addition text: splice into that addition. + if left and anchor_at(i1 - 1) is None and (found := find_add(a[i1 - 1])): + add_id, blocks = found + idx = blocks.index(a[i1 - 1]) + 1 + blocks[idx:idx] = new_blocks + edits().adds[add_id] = "\n\n".join(blocks) + elif right and anchor_at(i1) is None and (found := find_add(a[i1])): + add_id, blocks = found + idx = blocks.index(a[i1]) + blocks[idx:idx] = new_blocks + edits().adds[add_id] = "\n\n".join(blocks) + else: + # An inter-chunk gap: anchor on the neighboring original + # chunks (both, when both verify — the first live referrer + # wins at apply time). + after_h = anchor_at(i1) if right and verified(i1) else None + before_h = anchor_at(i1 - 1) if left and verified(i1 - 1) else None + if after_h is None and before_h is None: + return # no live anchor (drifted base): skip + add_id = "" + for h, field in ((after_h, "before"), (before_h, "after")): + if h is not None and (ce := le.chunks.get(h) if le else None): + add_id = add_id or getattr(ce, field) + if add_id and add_id in edits().adds: + edits().adds[add_id] += "\n\n" + "\n\n".join(new_blocks) + else: + add_id = secrets.token_hex(6) + edits().adds[add_id] = "\n\n".join(new_blocks) + if after_h is not None: + edits().chunks.setdefault(after_h, ChunkEdit()).before = add_id + if before_h is not None: + edits().chunks.setdefault(before_h, ChunkEdit()).after = add_id + changed = True + + def do_replace(i: int, new_blocks: list[str]) -> None: + nonlocal changed + h = anchor_at(i) + if h is None: + if not (found := find_add(a[i])): + return + add_id, blocks = found + blocks[blocks.index(a[i]) : blocks.index(a[i]) + 1] = new_blocks + edits().adds[add_id] = "\n\n".join(blocks) + else: + if not verified(i): + return + ce = le.chunks.get(h) if le else None + # The base shows the live patch when one exists, else the + # served text: splice the edit into its blocks, so the patch + # always covers the chunk's whole text (a patch may hold + # several blocks — a paragraph split). a[i] not in the blocks + # = the base doesn't reflect this chunk (drifted): skip. + base_text = ce.replace if ce is not None and ce.replace else served(h) + blocks = chunk_markdown(base_text) + if a[i] not in blocks: + return + blocks[blocks.index(a[i]) : blocks.index(a[i]) + 1] = new_blocks + if ce is None: + ce = edits().chunks.setdefault(h, ChunkEdit()) + ce.replace = "\n\n".join(blocks) + ce.drop = False + changed = True + + def emit(tag: str, i1: int, i2: int, j1: int, j2: int) -> None: + if tag == "delete": + for i in range(i1, i2): + do_delete(i) + elif tag == "insert": + do_insert(i1, list(b[j1:j2])) + elif i2 - i1 == 1: # replace of one block, possibly into several + do_replace(i1, list(b[j1:j2])) + else: # a grown region: pair positionally, insert the surplus + for k in range(i2 - i1): + do_replace(i1 + k, [b[j1 + k]]) + do_insert(i2, list(b[j1 + i2 - i1 : j2])) + + for tag, i1, i2, j1, j2 in SequenceMatcher( + None, a, b, autojunk=False + ).get_opcodes(): + if tag == "equal": + continue + if tag == "replace" and i2 - i1 > j2 - j1: + for sub in _refine_replace(a, i1, i2, b, j1, j2): + emit(*sub) + else: + emit(tag, i1, i2, j1, j2) + if not changed: return False - data.patches.setdefault(f"{path}:{lang}", []).append(patch) node.langs[lang] = True return True @@ -223,19 +407,15 @@ def set_title_translation(data: Data, node: Node, lang: str, title: str) -> bool def clear_translations(data: Data) -> None: """Drop all machine translations (``Data.trans``) and rebuild the - availability index (``node.langs``) from the surviving user patches — - patches alone make a language exist on a page. Pure data ops — the + availability index (``node.langs``) from the surviving user overrides — + overrides alone make a language exist on a page. Pure data ops — the caller wraps in a transaction and invalidates.""" data.trans.clear() - patch_langs: dict[str, set[str]] = {} - for key in data.patches: - path, _, lang = key.rpartition(":") - patch_langs.setdefault(path, set()).add(lang) def walk(nodes: dict[str, Node], prefix: str) -> None: for slug, node in nodes.items(): path = f"{prefix}/{slug}" if prefix else slug - node.langs = {lang: True for lang in patch_langs.get(path, ())} + node.langs = {lang: True for lang in data.overrides.get(path, ())} walk(node.children, path) walk(data.menu, "") diff --git a/pagerite/migrations.py b/pagerite/migrations.py index 9129adc..319a791 100644 --- a/pagerite/migrations.py +++ b/pagerite/migrations.py @@ -150,13 +150,12 @@ def migrate_v3(d: dict) -> None: Chunk keys are 9-byte blake3 digests; at this raw JSON level they are base64 strings (decoding into the structs restores ``bytes`` keys). - ``trans``/``patches`` start empty; the translator job fills them and - maintains the ``langs`` index as translations land. ``language``, - ``no_trans`` and ``langs`` need nothing — struct defaults cover them. + ``trans`` starts empty; the translator job fills it and maintains the + ``langs`` index as translations land. ``language``, ``no_trans`` and + ``langs`` need nothing — struct defaults cover them. """ store = d.setdefault("chunks", {}) d.setdefault("trans", {}) - patches = d.setdefault("patches", {}) def walk(nodes: dict) -> None: for node in nodes.values(): @@ -171,8 +170,3 @@ def migrate_v3(d: dict) -> None: walk(node.get("children") or {}) walk(d.get("menu") or {}) - # Article paths never carry a leading slash in keys (docs/migrate.md). - # The only path-keyed store starts empty here, so this is defensive - # for databases that went through a downgrade/upgrade cycle. - for key in [k for k in patches if k.startswith("/")]: - patches[key.lstrip("/")] = patches.pop(key) diff --git a/pagerite/translate.py b/pagerite/translate.py index 7560ab9..4e28883 100644 --- a/pagerite/translate.py +++ b/pagerite/translate.py @@ -39,12 +39,7 @@ from fastapi import WebSocket, WebSocketDisconnect from kanta import Kanta from pagerite import i18n -from pagerite.chunks import ( - chunk_key, - chunk_markdown, - join_chunks, - needs_translation, -) +from pagerite.chunks import chunk_key, chunk_markdown, needs_translation from pagerite.data import Data, Node, node_markdown, resolve, sorted_nodes from pagerite.markdown import has_h1 from pagerite.segments import Span, join, pure_prose, split @@ -488,7 +483,7 @@ class Dispatcher: def _block_contexts(self, lang: str, item: TransItem) -> list[str]: """The previous and next block of the served hybrid around a pending - chunk (current machine translation with user patches applied, so + chunk (current machine translation with user overrides applied, so human corrections propagate into fresh translations).""" chain = resolve(self.data.menu, item.path) node = chain[-1] if chain else None @@ -498,10 +493,7 @@ class Dispatcher: self.data.trans.get(h, {}).get(lang) or self.data.chunks.get(h, "") for h in node.chunks ] - hybrid = join_chunks(served) - for patch in self.data.patches.get(f"{item.path}:{lang}", []): - hybrid = i18n.apply_patch(hybrid, patch) - blocks = chunk_markdown(hybrid) + blocks = chunk_markdown(i18n.hybrid_markdown(self.data, node, item.path, lang)) i = node.chunks.index(item.key) # Map the chunk's served-list position onto the patched block list # (patches may merge, split or drop blocks).