From d3e2196c83d85c784635da1bb45c70eb516ebb9c Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Thu, 3 Sep 2026 03:54:27 +0000 Subject: [PATCH] Localization (#1) Implement comprehensive content localization, admin panels for editing each language, AI translation interface with automatic updates when base language version is changed. - SEO tags for all language URLs - Uses accept-language by default, ?lang=en overrides temporarily - User edits patched on top of translations - RTL language supportReviewed-on: https://git.zi.fi/LeoVasanko/pagerite/pulls/1 --- AGENTS.md | 9 +- docs/editing.md | 11 +- docs/localization.md | 474 ++++++++++++++++++++++++++ docs/migrate.md | 178 ++++++++++ frontend/src/AnalyticsView.vue | 36 +- frontend/src/BannerEditor.vue | 57 +++- frontend/src/ConnNote.vue | 21 ++ frontend/src/EditorShell.vue | 49 ++- frontend/src/LangSelect.vue | 147 ++++++++ frontend/src/LocalizationEditor.vue | 281 +++++++++++++++ frontend/src/PageEditor.vue | 277 ++++++++++++--- frontend/src/StructureEditor.vue | 102 +++++- frontend/src/StructureTree.vue | 53 ++- frontend/src/editorLang.js | 15 + frontend/src/langs.js | 57 ++++ frontend/src/pagerite.js | 101 ++++-- frontend/src/reconnect.js | 56 +++ frontend/src/swapdoc.js | 31 +- frontend/vite.config.js | 2 +- pagerite/__main__.py | 7 + pagerite/app.py | 382 +++++++++++++++++---- pagerite/chunks.py | 162 +++++++++ pagerite/data.py | 70 +++- pagerite/i18n.py | 276 +++++++++++++++ pagerite/markdown.py | 76 +++-- pagerite/migrations.py | 38 +++ pagerite/seed.py | 4 +- pagerite/segments.py | 506 ++++++++++++++++++++++++++++ pagerite/translate.py | 389 +++++++++++++++++++++ pagerite/views.py | 188 ++++++++--- pyproject.toml | 2 +- scripts/translator.py | 319 ++++++++++++++++++ 32 files changed, 4134 insertions(+), 242 deletions(-) create mode 100644 docs/localization.md create mode 100644 docs/migrate.md create mode 100644 frontend/src/ConnNote.vue create mode 100644 frontend/src/LangSelect.vue create mode 100644 frontend/src/LocalizationEditor.vue create mode 100644 frontend/src/editorLang.js create mode 100644 frontend/src/langs.js create mode 100644 frontend/src/reconnect.js create mode 100644 pagerite/chunks.py create mode 100644 pagerite/i18n.py create mode 100644 pagerite/segments.py create mode 100644 pagerite/translate.py create mode 100644 scripts/translator.py diff --git a/AGENTS.md b/AGENTS.md index 47df64c..5bee8d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,10 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `pagerite/` — Python backend package (hatchling build target). - `app.py` — FastAPI app and route registration. - `data.py` — msgspec Structs for the kanta database. + - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). + - `i18n.py` — language selection, translation assembly (chunks + patches) and translated-edit recording (user patches, per-language title overrides, refresh). + - `translate.py` — translator service protocol (msgspec structs), the connected-client `Dispatcher` (job pipeline, result validation) and pending/store core for the `/_translate/{key}` WebSocket (docs/localization.md); app.py only registers the route. + - `segments.py` — the translation round trip: fragments split into pure-prose wire segments (via markdown.make_md's verbatim parser; link- and formatting-carrying blocks stay whole, link/formatted texts inline, Markdown stripped) and translations spliced back by source offset, link/formatting markdown re-inserted at weight-mapped positions (docs/localization.md). - `migrations.py` — kanta migrations (`migrate_vN`); ALL schema/storage upgrades live here (raw state dict before struct decoding), never in the app lifespan: v1 moves legacy in-db file blobs to the on-disk store and rebuilds the legacy flat `pages` as the menu tree, v2 rewrites `/_f/{hash}.ext` image links to the extension-less form, backfills AVIF/WebP/JPEG derivatives on disk and drops the obsolete `version` field. - `markdown.py` — markdown-it-py renderer. - `views.py` — shared page layout and rendering; theme/user-font resolution across `THEME_DIRS` / `FONT_DIRS` (cwd, site, platform data roots, then built-in `pagerite/themes/`, see `docs/themes-and-assets.md`). @@ -21,8 +25,11 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `main.js` — Vue editor app entry. - `analytics-main.js` — analytics page entry (mounts `AnalyticsView` at `/_a`). - `pagerite.js` — public page entry. + - `editorLang.js` + `LangSelect.vue` — the editor shell's shared language selection and its selector component (page + structure tabs; drives the page preview while the panel is open, via `swapdoc.setLangOverride`). + - `reconnect.js` — shared WebSocket pacing for all sockets (staggered connect slots, stuck-CONNECTING watchdog, exponential backoff): bursts and rapid retries trip the browser's WebSocket throttling. - `assets/` — base CSS, Pygments styles, fonts. - `scripts/devserver.py` — dev server with auto reload (the user mostly uses this; avoid running the server yourself, ask the user to test). +- `scripts/translator.py` — Seed-X translator service client for the `/_translate/{key}` socket (reference client, runs in its own uv env via PEP 723); stays connected full time, unloads the model after 60 s idle and reloads on the next job. Server run by CLI entry point `uv run pagerite` (no auto reloads, build needed). Dev mode is `scripts/devserver.py` (auto reloads, no build needed). @@ -53,5 +60,5 @@ Server run by CLI entry point `uv run pagerite` (no auto reloads, build needed). - Keep dependencies minimal; add via `uv add` and mention it. - The public URL space belongs to content (pretty slugs at root). Reserve only `/_` for the machinery (`/_api/`, `/_f/`, `/_assets/`), plus `/favicon.ico` from the build. Slugs are lowercase ASCII letters, digits, hyphens and underscores `[a-z0-9_-]` (the site editor filters input live via `slugify.js`, built on the `transliteration` npm package — unicode folds to ASCII, spaces become hyphens; an empty slug on a new page is derived from its title), may not begin with `_` or `.`, and such URLs are never looked up as content. -- No auth in core code; the SSO/reverse proxy gates all of `/_api` (forward-auth) and owns `/auth/` (login/logout, session validation). Pages render identically for everyone; pagerite.js adds the editing UI only after the auth server validates the session. +- No auth in core code; the SSO/reverse proxy gates all of `/_api` (forward-auth) and owns `/auth/` (login/logout, session validation). Pages render identically for everyone; pagerite.js adds the editing UI only after the auth server validates the session. The one keyed exception is `/_translate/{key}` (translator service; `Data.translate_keys`, see docs/localization.md). - Update the relevant MarkDown files when architecture, tooling, or conventions change. diff --git a/docs/editing.md b/docs/editing.md index c3420dd..3ad6f68 100644 --- a/docs/editing.md +++ b/docs/editing.md @@ -4,17 +4,22 @@ The Vue editor is a single tabbed `EditorShell.vue` mounted in a host div create ## Tabs -The shell hosts four kept-alive tabs (ordered site-wide first — site, structure — then, after a visual break, the per-page tabs — article, banner): +The shell hosts five kept-alive tabs (ordered site-wide first — site, structure, localization — then, after a visual break, the per-page tabs — article, banner): - `PageEditor.vue` — CodeMirror + server-rendered preview over WebSocket `/_api/ws/editor`, previewing into the visible article; editor and article scrolls are linked piecewise-linearly, keyed on the section anchors' `data-line` (markdown source line the backend stamps on top-level anchored h1/h2s): the page follows the cursor (fractional, wrap-aware, scrolling only when the cursor's page position leaves the viewport, with an edge margin), the editor follows page scroll with a progress-based viewport anchor, applied instantly (the window keeps scrolling normally while any editor is open — the panel is fixed to the viewport's left edge, its top tracking the banner's bottom edge until the banner scrolls away — and the panel scrolls internally); anchored h2s carry their own edit pens that open the editor scrolled to that section; a format bar offers Markdown helpers — bold/italic/code/link/table/image upload (always block-level on a fresh blank-separated line of its own — a cursor on a non-empty line, e.g. inside an existing image tag, inserts after that line, never into it; always with an empty `""` caption, cursor inside the quotes), toggling fences (` ``` ` code blocks and `::: aside` containers share the same machinery: clicked inside one they remove it and select the content, otherwise they wrap the selection or the cursor's line, keeping it selected), and `.left`/`.right`/`.wide`/`.margin` placement toggles plus `.small`/`.large`/`.huge` text-size toggles (brace attributes on the block at the cursor, mutually exclusive within each group; on `:::` containers a placement class replaces the container name instead — `::: aside` → `::: margin`), with Ctrl/Cmd-B/I/S bindings — for the hard-to-remember syntax. Edits content and title only, never the path. - `BannerEditor.vue` — per-page banner HTML + banner design selector, previewed into `#page-banner`. - `SiteEditor.vue` — site brand + optional custom brand HTML with image/video upload + theme selector + page-transition selector + font picker + favicon upload — clicking the preview tile picks a new one — + site-wide custom CSS, CSS injected into ``. -- `StructureEditor.vue` — the vue-draggable structure tree with always-editable title/slug inputs per row. +- `StructureEditor.vue` — the vue-draggable structure tree with always-editable title/slug inputs per row, plus a per-row flag dropdown setting the page's primary language (`Node.language`, inherited by the subtree). +- `LocalizationEditor.vue` — the site-wide translation settings: target languages as a flag grid (toggles, grouped in geographic rows; see docs/localization.md), the refresh-all-translations button, and the translator service WebSocket URL(s) to connect `scripts/translator.py` to. Media uploads everywhere use the image icon buttons (pasting into the editor works too). The article, banner and site-settings pens are shorthands that open the shell on the matching tab; once open, clicking a pen switches tabs (and retargets the editors to the current page) instead of closing/remounting. The close button in the tab bar closes the shell (deliberately NOT Escape — it fired too easily by accident); tabs have no close buttons of their own. Closing only HIDES the shell — the Vue app stays mounted, so page-editor state (unsaved text included) survives until a real page reload; the editor always follows the URL, so fetch-navigating with the shell open (or before re-opening it) retargets it to the new page — unsaved text is stashed per path for the session and restored when returning, cleared on save. Saving there is explicit (Ctrl+S) and refreshes the page regions in place. Admin panels never reload the page. In-place page re-rendering shared by the banner/site/structure tabs lives in `swapdoc.js` (`runScripts`/`loadPlain`: fetch a page, swap the dynamic regions, replaceState). It also exports `dropPageCache`, which the editor tabs call after any save that can alter the rendered HTML of other pages (theme, headings, structure, banners, site brand/CSS, favicon). Dropping the cache while editing avoids re-fetching every page immediately; the public runtime re-preloads visible links once the editor panel closes. +The page and structure tabs share one language selector: `LangSelect.vue` (small flag + dropdown) v-modeled on the shell-wide selection in `editorLang.js` (`''` = primary). While the panel is open that selection overrides the page's normal language preferences: EditorShell calls `swapdoc.setLangOverride`, which pins every `loadPlain` fetch (`?lang=`, the primary by its own code) and pagerite.js's own fetches/prefetches (`pagerite:session-lang`), until the panel closes and the override clears. + +All WebSockets (page/banner editors, analytics view, the pagerite.js activity channel) pace their connections through `reconnect.js`: new sockets are created a staggered slot apart (a page load opens Vite's HMR socket plus several of ours at the same moment, and such bursts — like rapid retries — trip the browser's WebSocket throttling, leaving every socket to the host "pending" for minutes), a watchdog closes sockets stuck CONNECTING so they reschedule instead of hanging forever, and retries follow an exponential backoff with jitter that only a healthy connection resets. While a socket is connecting or waiting to reconnect the panel says so (`ConnNote.vue`), and the CodeMirror editors stay locked until their document arrives (typing before the doc accept would be clobbered by it). + ## Saving behavior Everything saves immediately as you edit (brand/title/CSS debounced, slug on commit since it renames the path), theme change swaps the stylesheet in place, tree rows navigate in place without transitions when focused, and the front page is a root-only row whose empty slug is editable like any other. Saves that can affect other pages drop the prefetch cache; the cache is rebuilt when the editor panel closes so navigation stays instant. @@ -25,6 +30,6 @@ Dropping ON the lower part of a row moves the page under that row (the child lis The shell is dynamic-imported onto the content page by pagerite.js when an edit pen is clicked (the pens are injected by pagerite.js after the session validates; they carry `data-editor-src`/`data-editor-css`/`data-editor-mode`). In dev, modules load from the Vite dev server (`PAGERITE_VITE_URL`), in prod from the hashed build assets resolved via `frontend-build/.vite/manifest.json`. -`vite.config.js` sets `appType: 'mpa'` (no SPA fallback) and builds with `manifest: true`, `assetsDir: '_/assets'` (so the build mirrors the URL space; `frontend/public/favicon.ico` lands at the build root and is served at `/favicon.ico`). JS inputs are `src/main.js` and `src/pagerite.js`, plus `src/assets/pagerite.css` as a separate stylesheet entry; theme, banner-design and transition CSS are NOT built — they live in `pagerite/themes/{name}/` and are served by the backend. There is no `index.html` source (it would shadow `/` and turn missing dev paths into an empty Vue shell). All outputs are ES modules. The build sets `preserveEntrySignatures: 'exports-only'` because main.js is consumed via dynamic `import()` for its `openEditor`/`closeEditor` exports — Vite app builds otherwise strip unused entry exports, leaving dead edit pens. In dev the backend links theme/banner-design stylesheets like in prod (`/_themes/...`); only the base CSS is Vite-injected from JS, and pagerite.js then re-appends the `#pagerite-theme`/`#pagerite-banner`/`#pagerite-transition`/`#pagerite-user` elements to restore the canonical order (base < theme < design < transition < custom CSS). In production all page assets are inlined instead (styles as ` diff --git a/frontend/src/EditorShell.vue b/frontend/src/EditorShell.vue index 064a23c..a53d152 100644 --- a/frontend/src/EditorShell.vue +++ b/frontend/src/EditorShell.vue @@ -1,5 +1,5 @@