From 2aed176c9e0743099740372e2a14ebd873e55deb Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Wed, 23 Sep 2026 07:30:32 +0000 Subject: [PATCH] Cache feed/llms.txt bodies, serve with content-hash ETags Rendering every article per request is expensive. Bodies are now RAM-cached keyed by the public base URL and cleared from _invalidate_pages (the same hook that drops the page render cache on any content/settings write). Responses carry a blake3 content-hash ETag and answer 304, so polling feed readers revalidate cheaply; matching page behavior, 304 revalidations are not recorded in analytics. --- docs/backend.md | 2 +- pagerite/feeds.py | 93 +++++++++++++++++++++++++++++++---------------- pagerite/state.py | 9 ++++- 3 files changed, 70 insertions(+), 34 deletions(-) diff --git a/docs/backend.md b/docs/backend.md index 5367bc4..4804fdf 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -10,7 +10,7 @@ Thin FastAPI assembly: lifespan (open the kanta database, load the file store, t - `files.py` — the `FileStore` and image derivative helpers, and the file routes: `/_api/files`, `/_f/`, `/_themes/`, `/_fonts/`, the favicon settings endpoints. - `api.py` — the editor REST API and WebSockets: `/_api/pages`, `/_api/structure`, `/_api/settings`, `/_api/toggle-task`, `/_api/translations`, `/_api/ws/editor`, and the translator channel `/_translate/{clientkey}`. - `tracking.py` — visit analytics: GeoIP, client enrichment, favicon fetching, debounced broadcasts, the `/_ws` activity socket, the admin stream `/_api/ws/analytics`, and the `/_a` viewer page. -- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), all carrying every published article with full content (relative URLs absolutized), linked from every page's `` and from the sitemap, and recorded in analytics like page GETs. +- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), all carrying every published article with full content (relative URLs absolutized), linked from every page's `` and from the sitemap, and recorded in analytics like page GETs. Bodies are RAM-cached (keyed by base URL, cleared by `_invalidate_pages` like the page render cache) and served with a content-hash ETag (304 revalidation). - `pages.py` — the public content pages: `/`, `/sitemap.xml`, `/robots.txt` and the `/{path:path}` catch-all. Route ordering is load-bearing and lives in `app.py`: the api/tracking/files routers are included BEFORE `frontend.route(app, "/")` is called — fastapi-vue inserts its file routes at the position where `route()` was called (during `load()` in the lifespan), so anything registered earlier wins. The content catch-all `/{path:path}` is included AFTER `frontend.route()` so that built frontend assets still take priority over content slugs. The `Frontend` is constructed with `spa=False` explicitly: it only serves the built files without a catch-all. diff --git a/pagerite/feeds.py b/pagerite/feeds.py index b9ec363..24a5692 100644 --- a/pagerite/feeds.py +++ b/pagerite/feeds.py @@ -11,11 +11,19 @@ All three are linked from every page's (see views._layout) and from the sitemap, recorded in analytics like page GETs (they surface as crawler hits — no activity message ever follows them), and rendered in the site's primary language only (feeds have no per-language negotiation here). + +Rendering every article is expensive, so bodies are cached in RAM keyed by +the public base URL and cleared by ``state._invalidate_pages`` on any +content/settings write — the same hook that drops the page render cache. +Each response also carries a content-hash ETag and answers 304, so polling +feed readers revalidate cheaply. """ +import blake3 import json from datetime import UTC from email.utils import format_datetime +from functools import lru_cache from xml.sax.saxutils import escape as xml_escape from fastapi import APIRouter, Request @@ -67,28 +75,17 @@ def _iso(node: Node) -> str: return node.modified.astimezone(UTC).replace(microsecond=0).isoformat() -@router.get("/llms.txt") -async def llms_txt(request: Request) -> Response: - """Markdown map of the site for LLM agents (llmstxt.org).""" - base = SITE_URL or str(request.base_url).rstrip("/") +def _render_llms(base: str) -> str: lines = [f"# {data.brand}", "", "## Pages", ""] for path, node in _articles(): url = f"{base}/{path}" if path else base excerpt = _description(_body_html(node, base), 120) suffix = f": {excerpt}" if excerpt else "" lines.append(f"- [{node.title or path}]({url}){suffix}") - _record_get(request) - return Response( - "\n".join(lines) + "\n", - media_type="text/plain", - headers={"cache-control": "no-cache"}, - ) + return "\n".join(lines) + "\n" -@router.get("/feed.json") -async def feed_json(request: Request) -> Response: - """JSON Feed 1.1 of all published articles, full content.""" - base = SITE_URL or str(request.base_url).rstrip("/") +def _render_feed_json(base: str) -> str: items = [ { "id": (url := f"{base}/{path}" if path else base), @@ -109,19 +106,10 @@ async def feed_json(request: Request) -> Response: "feed_url": f"{base}/feed.json", "items": items, } - _record_get(request) - return Response( - json.dumps(feed, ensure_ascii=False, indent=1), - media_type="application/feed+json", - headers={"cache-control": "no-cache"}, - ) + return json.dumps(feed, ensure_ascii=False, indent=1) -@router.get("/feed.xml") -async def feed_xml(request: Request) -> Response: - """RSS 2.0 of all published articles (full content in CDATA), with an - atom:link self reference.""" - base = SITE_URL or str(request.base_url).rstrip("/") +def _render_feed_xml(base: str) -> str: items = [] for path, node in _articles(): url = f"{base}/{path}" if path else base @@ -133,7 +121,7 @@ async def feed_xml(request: Request) -> Response: f"" f"" ) - xml = ( + return ( '' '' f"{xml_escape(data.brand)}" @@ -143,9 +131,52 @@ async def feed_xml(request: Request) -> Response: + "".join(items) + "" ) + + +#: Export body builders by route path. +_RENDERERS = { + "/llms.txt": (_render_llms, "text/plain"), + "/feed.json": (_render_feed_json, "application/feed+json"), + "/feed.xml": (_render_feed_xml, "application/rss+xml"), +} + + +@lru_cache(maxsize=12) +def _cached_feed(path: str, base_url: str) -> str: + """Rendered export body; cleared by ``state._invalidate_pages`` on any + content/settings write. base_url is part of the key because the bodies + bake absolute URLs into every link, image and guid.""" + return _RENDERERS[path][0](base_url) + + +def _feed_response(request: Request, path: str) -> Response: + """Cached export response with a content-hash ETag (304 on match), so + polling feed readers revalidate without a rerender or a download.""" + base = SITE_URL or str(request.base_url).rstrip("/") + body = _cached_feed(path, base) + headers = {"cache-control": "no-cache"} + tag = f'"{blake3.blake3(body.encode()).hexdigest()[:32]}"' + headers["etag"] = tag + if request.headers.get("if-none-match") == tag: + return Response(status_code=304, headers=headers) _record_get(request) - return Response( - xml, - media_type="application/rss+xml", - headers={"cache-control": "no-cache"}, - ) + return Response(body, media_type=_RENDERERS[path][1], headers=headers) + + +@router.get("/llms.txt") +async def llms_txt(request: Request) -> Response: + """Markdown map of the site for LLM agents (llmstxt.org).""" + return _feed_response(request, "/llms.txt") + + +@router.get("/feed.json") +async def feed_json(request: Request) -> Response: + """JSON Feed 1.1 of all published articles, full content.""" + return _feed_response(request, "/feed.json") + + +@router.get("/feed.xml") +async def feed_xml(request: Request) -> Response: + """RSS 2.0 of all published articles (full content in CDATA), with an + atom:link self reference.""" + return _feed_response(request, "/feed.xml") diff --git a/pagerite/state.py b/pagerite/state.py index 9ed847f..5c8eb71 100644 --- a/pagerite/state.py +++ b/pagerite/state.py @@ -171,11 +171,16 @@ _render_gen = 0 def _invalidate_pages() -> None: - """Drop cached page bodies and bump the render generation (ETags); - any content change also re-runs translation dispatch.""" + """Drop cached page bodies (and the feed/llms.txt export bodies) and + bump the render generation (ETags); any content change also re-runs + translation dispatch.""" global _render_gen _render_gen += 1 _cached_body.cache_clear() + # Local import: pagerite.feeds imports this module. + from pagerite import feeds + + feeds._cached_feed.cache_clear() dispatcher.schedule()