Cache feed/llms.txt bodies, serve with content-hash ETags

Rendering every article per request is expensive. Bodies are now RAM-cached keyed by the public base URL and cleared from _invalidate_pages (the same hook that drops the page render cache on any content/settings write). Responses carry a blake3 content-hash ETag and answer 304, so polling feed readers revalidate cheaply; matching page behavior, 304 revalidations are not recorded in analytics.
2026-09-23 07:30:32 +00:00
parent 9ff22016d1
commit 2aed176c9e
3 changed files with 70 additions and 34 deletions
@@ -10,7 +10,7 @@ Thin FastAPI assembly: lifespan (open the kanta database, load the file store, t
 - `files.py` — the `FileStore` and image derivative helpers, and the file routes: `/_api/files`, `/_f/`, `/_themes/`, `/_fonts/`, the favicon settings endpoints.
 - `api.py` — the editor REST API and WebSockets: `/_api/pages`, `/_api/structure`, `/_api/settings`, `/_api/toggle-task`, `/_api/translations`, `/_api/ws/editor`, and the translator channel `/_translate/{clientkey}`.
 - `tracking.py` — visit analytics: GeoIP, client enrichment, favicon fetching, debounced broadcasts, the `/_ws` activity socket, the admin stream `/_api/ws/analytics`, and the `/_a` viewer page.
- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), all carrying every published article with full content (relative URLs absolutized), linked from every page's `<head>` and from the sitemap, and recorded in analytics like page GETs.
+- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), all carrying every published article with full content (relative URLs absolutized), linked from every page's `<head>` and from the sitemap, and recorded in analytics like page GETs. Bodies are RAM-cached (keyed by base URL, cleared by `_invalidate_pages` like the page render cache) and served with a content-hash ETag (304 revalidation).
 - `pages.py` — the public content pages: `/`, `/sitemap.xml`, `/robots.txt` and the `/{path:path}` catch-all.
 Route ordering is load-bearing and lives in `app.py`: the api/tracking/files routers are included BEFORE `frontend.route(app, "/")` is called — fastapi-vue inserts its file routes at the position where `route()` was called (during `load()` in the lifespan), so anything registered earlier wins. The content catch-all `/{path:path}` is included AFTER `frontend.route()` so that built frontend assets still take priority over content slugs. The `Frontend` is constructed with `spa=False` explicitly: it only serves the built files without a catch-all.
@@ -11,11 +11,19 @@ All three are linked from every page's <head> (see views._layout) and from
 the sitemap, recorded in analytics like page GETs (they surface as crawler
 hits — no activity message ever follows them), and rendered in the site's
 primary language only (feeds have no per-language negotiation here).
 Rendering every article is expensive, so bodies are cached in RAM keyed by
 the public base URL and cleared by ``state._invalidate_pages`` on any
 content/settings write — the same hook that drops the page render cache.
 Each response also carries a content-hash ETag and answers 304, so polling
 feed readers revalidate cheaply.
 """
 import blake3
 import json
 from datetime import UTC
 from email.utils import format_datetime
 from functools import lru_cache
 from xml.sax.saxutils import escape as xml_escape
 from fastapi import APIRouter, Request
@@ -67,28 +75,17 @@ def _iso(node: Node) -> str:
    return node.modified.astimezone(UTC).replace(microsecond=0).isoformat()
-@router.get("/llms.txt")
+def _render_llms(base: str) -> str:
 async def llms_txt(request: Request) -> Response:
    """Markdown map of the site for LLM agents (llmstxt.org)."""
    base = SITE_URL or str(request.base_url).rstrip("/")
    lines = [f"# {data.brand}", "", "## Pages", ""]
    for path, node in _articles():
        url = f"{base}/{path}" if path else base
        excerpt = _description(_body_html(node, base), 120)
        suffix = f": {excerpt}" if excerpt else ""
        lines.append(f"- [{node.title or path}]({url}){suffix}")
-    _record_get(request)
+    return "\n".join(lines) + "\n"
    return Response(
        "\n".join(lines) + "\n",
        media_type="text/plain",
        headers={"cache-control": "no-cache"},
    )
-@router.get("/feed.json")
+def _render_feed_json(base: str) -> str:
 async def feed_json(request: Request) -> Response:
    """JSON Feed 1.1 of all published articles, full content."""
    base = SITE_URL or str(request.base_url).rstrip("/")
    items = [
        {
            "id": (url := f"{base}/{path}" if path else base),
@@ -109,19 +106,10 @@ async def feed_json(request: Request) -> Response:
        "feed_url": f"{base}/feed.json",
        "items": items,
    }
-    _record_get(request)
+    return json.dumps(feed, ensure_ascii=False, indent=1)
    return Response(
        json.dumps(feed, ensure_ascii=False, indent=1),
        media_type="application/feed+json",
        headers={"cache-control": "no-cache"},
    )
-@router.get("/feed.xml")
+def _render_feed_xml(base: str) -> str:
 async def feed_xml(request: Request) -> Response:
    """RSS 2.0 of all published articles (full content in CDATA), with an
    atom:link self reference."""
    base = SITE_URL or str(request.base_url).rstrip("/")
    items = []
    for path, node in _articles():
        url = f"{base}/{path}" if path else base
@@ -133,7 +121,7 @@ async def feed_xml(request: Request) -> Response:
            f"<description><![CDATA[{_body_html(node, base)}]]></description>"
            f"</item>"
        )
-    xml = (
+    return (
        '<?xml version="1.0" encoding="UTF-8"?>'
        '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">'
        f"<channel><title>{xml_escape(data.brand)}</title>"
@@ -143,9 +131,52 @@ async def feed_xml(request: Request) -> Response:
        + "".join(items)
        + "</channel></rss>"
    )
 #: Export body builders by route path.
 _RENDERERS = {
    "/llms.txt": (_render_llms, "text/plain"),
    "/feed.json": (_render_feed_json, "application/feed+json"),
    "/feed.xml": (_render_feed_xml, "application/rss+xml"),
 }
@lru_cache(maxsize=12)
 def _cached_feed(path: str, base_url: str) -> str:
    """Rendered export body; cleared by ``state._invalidate_pages`` on any
    content/settings write. base_url is part of the key because the bodies
    bake absolute URLs into every link, image and guid."""
    return _RENDERERS[path][0](base_url)
 def _feed_response(request: Request, path: str) -> Response:
    """Cached export response with a content-hash ETag (304 on match), so
    polling feed readers revalidate without a rerender or a download."""
    base = SITE_URL or str(request.base_url).rstrip("/")
    body = _cached_feed(path, base)
    headers = {"cache-control": "no-cache"}
    tag = f'"{blake3.blake3(body.encode()).hexdigest()[:32]}"'
    headers["etag"] = tag
    if request.headers.get("if-none-match") == tag:
        return Response(status_code=304, headers=headers)
    _record_get(request)
-    return Response(
+    return Response(body, media_type=_RENDERERS[path][1], headers=headers)
-        xml,
+
-        media_type="application/rss+xml",
+
-        headers={"cache-control": "no-cache"},
+@router.get("/llms.txt")
-    )
+async def llms_txt(request: Request) -> Response:
    """Markdown map of the site for LLM agents (llmstxt.org)."""
    return _feed_response(request, "/llms.txt")
@router.get("/feed.json")
 async def feed_json(request: Request) -> Response:
    """JSON Feed 1.1 of all published articles, full content."""
    return _feed_response(request, "/feed.json")
@router.get("/feed.xml")
 async def feed_xml(request: Request) -> Response:
    """RSS 2.0 of all published articles (full content in CDATA), with an
    atom:link self reference."""
    return _feed_response(request, "/feed.xml")
@@ -171,11 +171,16 @@ _render_gen = 0
 def _invalidate_pages() -> None:
-    """Drop cached page bodies and bump the render generation (ETags);
+    """Drop cached page bodies (and the feed/llms.txt export bodies) and
-    any content change also re-runs translation dispatch."""
+    bump the render generation (ETags); any content change also re-runs
    translation dispatch."""
    global _render_gen
    _render_gen += 1
    _cached_body.cache_clear()
    # Local import: pagerite.feeds imports this module.
    from pagerite import feeds
    feeds._cached_feed.cache_clear()
    dispatcher.schedule()