Encapsulate all migrations in kanta migrate_vN, drop persisted Data.version
migrate_v1 now also rebuilds the legacy flat pages store as the menu tree (moved from the app lifespan, raw-dict level); migrate_v2 now also backfills missing AVIF/WebP/JPEG derivatives on disk (moved from the lifespan) and drops the obsolete version field. The version render counter was cache-invalidation state, not database state: replaced by an in-memory render generation that clears the page-body LRU and feeds page ETags. The legacy Page struct and Data.pages/version fields are removed; old databases lose the stale keys on re-serialization.
This commit is contained in:
+31
-77
@@ -261,24 +261,6 @@ def _remove_page_content(menu: dict[str, Node], path: str) -> None:
|
||||
del slot[0][slot[1]]
|
||||
|
||||
|
||||
def _migrate_legacy() -> None:
|
||||
"""Rebuild the legacy flat page store as a tree (one-time migration)."""
|
||||
if not data.pages:
|
||||
return
|
||||
with kanta.transaction("migrate pages to tree"):
|
||||
for path, page in data.pages.items():
|
||||
node = _ensure(data.menu, path)
|
||||
node.title = page.title
|
||||
node.content = page.markdown
|
||||
node.banner = page.banner
|
||||
node.published = page.published
|
||||
node.order = page.order
|
||||
node.created = page.created
|
||||
node.modified = page.modified
|
||||
data.pages.clear()
|
||||
data.version += 1
|
||||
|
||||
|
||||
@kanta.bootstrap
|
||||
def _seed(data: Data) -> None:
|
||||
"""Write the demo pages on database creation (never on existing dbs)."""
|
||||
@@ -298,52 +280,11 @@ def _seed(data: Data) -> None:
|
||||
node.order = order
|
||||
|
||||
|
||||
def _backfill_derivatives() -> None:
|
||||
"""Create missing AVIF/WebP/JPEG derivatives for files stored before
|
||||
they were introduced (older uploads may have only the original plus
|
||||
AVIF, and SVGs no raster variants at all). WebP/JPEG are re-encoded
|
||||
from an existing AVIF when available, everything else from the
|
||||
original (SVGs rasterized first)."""
|
||||
try:
|
||||
paths = [f for f in file_store.path.iterdir() if f.is_file()]
|
||||
except FileNotFoundError:
|
||||
return
|
||||
groups: dict[str, list[Path]] = {}
|
||||
for p in paths:
|
||||
groups.setdefault(p.name.partition(".")[0], []).append(p)
|
||||
for digest, files in groups.items():
|
||||
names = {p.name for p in files}
|
||||
source = next(
|
||||
(p for p in files if ".orig." in p.name or p.suffix == ".svg"), None
|
||||
)
|
||||
if source is None:
|
||||
continue # plain as-is file, no derivatives to make
|
||||
avif = file_store.get(f"{digest}.avif")
|
||||
if avif is None:
|
||||
ext = source.suffix
|
||||
body = source.read_bytes()
|
||||
if ext == ".svg":
|
||||
png = _svg_to_png(body, IMAGE_MAXSIZE)
|
||||
if png is None:
|
||||
continue
|
||||
body, ext = png, ".png"
|
||||
converted = _to_avif(body, ext)
|
||||
if converted is None:
|
||||
continue
|
||||
file_store.put(f"{digest}.avif", converted)
|
||||
avif = file_store.get(f"{digest}.avif")
|
||||
for fmt, quality in (("webp", IMAGE_WEBP_QUALITY), ("jpg", IMAGE_JPG_QUALITY)):
|
||||
if f"{digest}.{fmt}" not in names:
|
||||
file_store.put(f"{digest}.{fmt}", _avif_to_format(avif[0], f".{fmt}", quality))
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||
"""Open the database, migrate legacy content, load assets, load GeoIP."""
|
||||
"""Open the database (migrations run inside kanta.open), load assets, load GeoIP."""
|
||||
await kanta.open()
|
||||
await asyncio.to_thread(file_store.load)
|
||||
await asyncio.to_thread(_backfill_derivatives)
|
||||
_migrate_legacy()
|
||||
await frontend.load()
|
||||
# Decompress/open the DB-IP MMDB once at startup. Lookups are then
|
||||
# read-only and safe to run in background ``to_thread`` workers.
|
||||
@@ -451,12 +392,25 @@ def _render_html(kind: str, path: str, base_url: str) -> str:
|
||||
return views.render_analytics(data.menu, data.brand, data.custom_css, data.theme, data.favicon, data.brand_html, transition=data.transition)
|
||||
|
||||
|
||||
# Render generation: bumped (and the body cache cleared) by every
|
||||
# content/settings write, so page ETags and cached copies invalidate when
|
||||
# navigation-affecting changes happen. In-memory only — not database state.
|
||||
_render_gen = 0
|
||||
|
||||
|
||||
def _invalidate_pages() -> None:
|
||||
"""Drop cached page bodies and bump the render generation (ETags)."""
|
||||
global _render_gen
|
||||
_render_gen += 1
|
||||
_cached_body.cache_clear()
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def _cached_body(kind: str, path: str, base_url: str, version: int, zstd: bool) -> bytes:
|
||||
"""Rendered page body. Every input the output depends on is in the key:
|
||||
data.version bumps on any content/settings change, base_url feeds the
|
||||
social meta URLs, and zstd selects the stored encoding (both variants
|
||||
are cached rather than re-compressed).
|
||||
def _cached_body(kind: str, path: str, base_url: str, zstd: bool) -> bytes:
|
||||
"""Rendered page body; cleared by _invalidate_pages on any
|
||||
content/settings change. base_url feeds the social meta URLs and zstd
|
||||
selects the stored encoding (both variants are cached rather than
|
||||
re-compressed).
|
||||
"""
|
||||
body = _render_html(kind, path, base_url).encode()
|
||||
return _zstd.compress(body) if zstd else body
|
||||
@@ -493,8 +447,8 @@ def _html_response(
|
||||
identity = _render_html(kind, path, base_url).encode()
|
||||
body = _zstd.compress(identity) if zstd else identity
|
||||
else:
|
||||
identity = _cached_body(kind, path, base_url, data.version, False)
|
||||
body = _cached_body(kind, path, base_url, data.version, True) if zstd else identity
|
||||
identity = _cached_body(kind, path, base_url, False)
|
||||
body = _cached_body(kind, path, base_url, True) if zstd else identity
|
||||
h = dict(headers or {})
|
||||
if zstd:
|
||||
h["vary"] = "accept-encoding"
|
||||
@@ -562,7 +516,7 @@ async def save_page(path: str, page: PageIn) -> None:
|
||||
if page.banner is not None:
|
||||
node.banner = page.banner
|
||||
node.modified = datetime.now(UTC)
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
|
||||
|
||||
class StructureOp(BaseModel):
|
||||
@@ -623,7 +577,7 @@ async def update_structure(op: StructureOp) -> None:
|
||||
elif op.order is not None:
|
||||
node.order = op.order
|
||||
node.modified = datetime.now(UTC)
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
|
||||
|
||||
@app.get("/_api/settings")
|
||||
@@ -657,14 +611,14 @@ class SettingsIn(BaseModel):
|
||||
|
||||
@app.put("/_api/settings", status_code=204)
|
||||
async def put_settings(settings: SettingsIn) -> None:
|
||||
"""Update site-wide settings; bumps the version so ETags invalidate."""
|
||||
"""Update site-wide settings; invalidates cached pages and ETags."""
|
||||
with kanta.transaction("update settings"):
|
||||
data.brand = settings.brand
|
||||
data.brand_html = settings.brand_html
|
||||
data.theme = settings.theme
|
||||
data.custom_css = settings.custom_css
|
||||
data.transition = settings.transition
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
|
||||
|
||||
@app.put("/_api/settings/favicon")
|
||||
@@ -694,7 +648,7 @@ async def put_favicon(request: Request) -> dict[str, str]:
|
||||
file_store.put(f"{digest}.{fmt}", variant)
|
||||
with kanta.transaction("upload favicon"):
|
||||
data.favicon = stored
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
return {"path": f"/_f/{stored}"}
|
||||
|
||||
|
||||
@@ -706,7 +660,7 @@ async def delete_favicon() -> None:
|
||||
"""
|
||||
with kanta.transaction("clear favicon"):
|
||||
data.favicon = ""
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
|
||||
|
||||
class ToggleTaskIn(BaseModel):
|
||||
@@ -743,7 +697,7 @@ async def toggle_task_endpoint(body: ToggleTaskIn) -> dict[str, str]:
|
||||
with kanta.transaction("toggle task", extra=path):
|
||||
node.content = new_markdown
|
||||
node.modified = datetime.now(UTC)
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
return {"markdown": new_markdown}
|
||||
|
||||
|
||||
@@ -975,7 +929,7 @@ async def delete_page(path: str) -> None:
|
||||
node.modified = datetime.now(UTC)
|
||||
else:
|
||||
del slot[0][slot[1]]
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
|
||||
|
||||
_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
|
||||
@@ -1428,7 +1382,7 @@ async def editor_ws(ws: WebSocket) -> None:
|
||||
if "banner_design" in msg:
|
||||
node.banner_design = msg["banner_design"]
|
||||
node.modified = datetime.now(UTC)
|
||||
data.version += 1
|
||||
_invalidate_pages()
|
||||
await ws.send_json({"type": "saved", "path": path})
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
@@ -1552,7 +1506,7 @@ async def show_page(request: Request, path: str) -> Response:
|
||||
# from pagerite.js's in-memory page cache (preload everything, never
|
||||
# fetch on navigation); the ETag just makes those one-time preload
|
||||
# fetches and any revalidation cheap.
|
||||
etag = f'"{path}@{node.modified.timestamp()}v{data.version}"'
|
||||
etag = f'"{path}@{node.modified.timestamp()}g{_render_gen}"'
|
||||
if request.headers.get("if-none-match") == etag:
|
||||
return Response(status_code=304)
|
||||
if _is_trackable_path(path):
|
||||
|
||||
@@ -50,34 +50,11 @@ class Node(msgspec.Struct, omit_defaults=True):
|
||||
)
|
||||
|
||||
|
||||
class Page(msgspec.Struct, omit_defaults=True):
|
||||
"""Legacy flat page record, from before the tree model.
|
||||
|
||||
Kept only so old databases still decode; app.py migrates any entries
|
||||
into ``Data.menu`` on startup and clears this.
|
||||
"""
|
||||
|
||||
title: str
|
||||
markdown: str
|
||||
published: bool = True
|
||||
order: float = 0
|
||||
banner: str = ""
|
||||
created: datetime = msgspec.field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
)
|
||||
modified: datetime = msgspec.field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class Data(msgspec.Struct):
|
||||
"""Root object of the kanta database. Owned and edited in place by us."""
|
||||
|
||||
#: Top-level menu items by slug; "" is the front page.
|
||||
menu: dict[str, Node] = {}
|
||||
#: Bumped on every structure/content write, so page ETags (which embed
|
||||
#: it) invalidate cached copies when navigation-affecting changes happen.
|
||||
version: int = 0
|
||||
#: Site name shown in the header and <title> suffix; editable in the
|
||||
#: site editor. Empty = no brand link in the header, no title suffix.
|
||||
brand: str = "Pagerite"
|
||||
@@ -101,9 +78,6 @@ class Data(msgspec.Struct):
|
||||
#: linked as <link rel="icon"> on every page. Empty = the build's
|
||||
#: /favicon.ico.
|
||||
favicon: str = ""
|
||||
#: Legacy flat page store (pre-tree databases); migrated into `menu`
|
||||
#: on startup, then cleared. Never written otherwise.
|
||||
pages: dict[str, Page] = {}
|
||||
|
||||
|
||||
def prettify(slug: str) -> str:
|
||||
|
||||
+104
-21
@@ -1,50 +1,133 @@
|
||||
"""Kanta schema migrations, discovered by name (``migrate_vN``).
|
||||
|
||||
Each function receives the raw state dict (JSON-level: bytes are base64
|
||||
strings) before it is decoded into ``Data`` structs, and runs exactly once
|
||||
strings, datetimes RFC 3339 strings, struct fields with default values
|
||||
omitted) before it is decoded into ``Data`` structs, and runs exactly once
|
||||
per database based on its recorded version.
|
||||
|
||||
All storage/schema upgrades live here — including on-disk file work, which
|
||||
runs through app.py's file store (imported lazily: app.py owns the store
|
||||
and passes this module to Kanta; at migration time, during lifespan
|
||||
``kanta.open()``, the app module is fully loaded).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
#: Extension-less file links: uploaded images are now linked as /_f/<hash>
|
||||
#: and the server negotiates avif/webp from the Accept header.
|
||||
_DERIVATIVE_LINK = re.compile(r"(/_f/[0-9a-f]{12})\.(?:avif|webp)\b")
|
||||
from pagerite.data import prettify
|
||||
|
||||
|
||||
def _rewrite_links(text: str | None) -> str | None:
|
||||
return None if text is None else _DERIVATIVE_LINK.sub(r"\1", text)
|
||||
def _append_order(nodes: dict) -> float:
|
||||
"""Raw-dict equivalent of data.append_order (order keys may be absent)."""
|
||||
return max((n.get("order", 0) for n in nodes.values()), default=0) + 1
|
||||
|
||||
|
||||
def _ensure(menu: dict, path: str) -> dict:
|
||||
"""Raw-dict equivalent of app._ensure: the node dict at ``path``,
|
||||
creating it and any missing ancestors (content-less category labels)
|
||||
appended at the end of their level."""
|
||||
nodes = menu
|
||||
node = None
|
||||
for seg in path.split("/"):
|
||||
node = nodes.get(seg)
|
||||
if node is None:
|
||||
node = {"title": prettify(seg), "order": _append_order(nodes)}
|
||||
nodes[seg] = node
|
||||
nodes = node.setdefault("children", {})
|
||||
return node
|
||||
|
||||
|
||||
def migrate_v1(d: dict) -> None:
|
||||
"""Move in-database file blobs to the on-disk content-addressed store."""
|
||||
"""Move in-database file blobs to the on-disk content-addressed store,
|
||||
and rebuild the legacy flat page store (``pages``) as the menu tree."""
|
||||
files = d.pop("files", None)
|
||||
if not files:
|
||||
if files:
|
||||
from pagerite.app import file_store
|
||||
|
||||
for name, body in files.items():
|
||||
if isinstance(body, str): # JSON-level bytes are base64 strings
|
||||
body = base64.b64decode(body)
|
||||
file_store.put(name, body)
|
||||
pages = d.pop("pages", None)
|
||||
if not pages:
|
||||
return
|
||||
# Deferred import: app.py owns the file store and passes this module to
|
||||
# Kanta; at migration time (lifespan open) the module is fully loaded.
|
||||
from pagerite.app import file_store
|
||||
|
||||
for name, body in files.items():
|
||||
if isinstance(body, str): # JSON-level bytes are base64 strings
|
||||
body = base64.b64decode(body)
|
||||
file_store.put(name, body)
|
||||
menu = d.setdefault("menu", {})
|
||||
for path, page in pages.items():
|
||||
node = _ensure(menu, path)
|
||||
node["title"] = page["title"]
|
||||
node["content"] = page["markdown"]
|
||||
for key in ("banner", "published", "order", "created", "modified"):
|
||||
if key in page:
|
||||
node[key] = page[key]
|
||||
|
||||
|
||||
def _rewrite_links(text: str) -> str:
|
||||
return _DERIVATIVE_LINK.sub(r"\1", text)
|
||||
#: Extension-less file links: uploaded images are linked as /_f/<hash>
|
||||
#: and the server negotiates avif/webp/jpg from the Accept header.
|
||||
_DERIVATIVE_LINK = re.compile(r"(/_f/[0-9a-f]{12})\.(?:avif|webp)\b")
|
||||
|
||||
|
||||
def _backfill_derivatives() -> None:
|
||||
"""Create missing AVIF/WebP/JPEG derivatives for files stored before
|
||||
they were introduced (older uploads may have only the original plus
|
||||
AVIF, and SVGs no raster variants at all). WebP/JPEG are re-encoded
|
||||
from an existing AVIF when available, everything else from the
|
||||
original (SVGs rasterized first)."""
|
||||
from pagerite import app
|
||||
|
||||
file_store = app.file_store
|
||||
try:
|
||||
paths = [f for f in file_store.path.iterdir() if f.is_file()]
|
||||
except FileNotFoundError:
|
||||
return
|
||||
groups: dict[str, list[Path]] = {}
|
||||
for p in paths:
|
||||
groups.setdefault(p.name.partition(".")[0], []).append(p)
|
||||
for digest, files in groups.items():
|
||||
names = {p.name for p in files}
|
||||
source = next(
|
||||
(p for p in files if ".orig." in p.name or p.suffix == ".svg"), None
|
||||
)
|
||||
if source is None:
|
||||
continue # plain as-is file, no derivatives to make
|
||||
avif = file_store.get(f"{digest}.avif")
|
||||
if avif is None:
|
||||
ext = source.suffix
|
||||
body = source.read_bytes()
|
||||
if ext == ".svg":
|
||||
png = app._svg_to_png(body, app.IMAGE_MAXSIZE)
|
||||
if png is None:
|
||||
continue
|
||||
body, ext = png, ".png"
|
||||
converted = app._to_avif(body, ext)
|
||||
if converted is None:
|
||||
continue
|
||||
file_store.put(f"{digest}.avif", converted)
|
||||
avif = file_store.get(f"{digest}.avif")
|
||||
for fmt, quality in (
|
||||
("webp", app.IMAGE_WEBP_QUALITY),
|
||||
("jpg", app.IMAGE_JPG_QUALITY),
|
||||
):
|
||||
if f"{digest}.{fmt}" not in names:
|
||||
file_store.put(
|
||||
f"{digest}.{fmt}", app._avif_to_format(avif[0], f".{fmt}", quality)
|
||||
)
|
||||
|
||||
|
||||
def migrate_v2(d: dict) -> None:
|
||||
"""Strip .avif/.webp extensions from /_f/ links in page content and
|
||||
banners (extension-less URLs negotiate the format by Accept header)."""
|
||||
"""Extension-less image links: strip .avif/.webp extensions from /_f/
|
||||
links in page content and banners (the server now negotiates the format
|
||||
by Accept header), backfill missing AVIF/WebP/JPEG derivatives on disk,
|
||||
and drop the obsolete render-counter field ``version`` (invalidation is
|
||||
an in-memory concern now, not database state)."""
|
||||
|
||||
def walk(nodes: dict) -> None:
|
||||
for node in nodes.values():
|
||||
for field in ("content", "banner"):
|
||||
if isinstance(node.get(field), str):
|
||||
node[field] = _rewrite_links(node[field])
|
||||
node[field] = _DERIVATIVE_LINK.sub(r"\1", node[field])
|
||||
walk(node.get("children") or {})
|
||||
|
||||
walk(d.get("menu") or {})
|
||||
d.pop("version", None)
|
||||
_backfill_derivatives()
|
||||
|
||||
Reference in New Issue
Block a user