diff --git a/AGENTS.md b/AGENTS.md index f5d6662..cf26734 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `api.py` — editor REST + WS: `/_api/pages`, `/_api/structure`, `/_api/settings`, `/_api/toggle-task`, `/_api/translations`, `/_api/ws/editor`, `/_translate/{key}`. - `tracking.py` — visit analytics: GeoIP, client enrichment, favicon fetch, `/_ws`, `/_api/ws/analytics`, the `/_a` page (docs/analytics.md). - `pages.py` — public content pages: `/`, `/sitemap.xml`, `/robots.txt`, the `/{path:path}` catch-all. + - `feeds.py` — machine-readable exports: `/llms.txt`, `/feed.json` (JSON Feed 1.1), `/feed.xml` (RSS 2.0 + atom:link); all published articles, full content, linked from every page `` and the sitemap, recorded in analytics. - `data.py` — msgspec Structs for the kanta database. - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). - `i18n.py` — language selection, translation assembly (chunks + overrides) and translated-edit recording (per-chunk user overrides in `Data.overrides`, per-language title overrides, refresh). diff --git a/docs/analytics.md b/docs/analytics.md index f2a7078..3e4284c 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -213,9 +213,9 @@ for misses. are caught by the abuse rules regardless. In the viewer, crawler hits are grouped by client hash and shown as a trail of pages, preceded by the referer when there is one (rendered with its favicon like visit - referers). Non-article machinery GETs (`/robots.txt`, `/sitemap.xml` — - and feed paths such as `/rss.xml` once those routes exist) appear as - emoji-marked steps (🤖 / 🗺️ / 📡) so they stand out from article steps. + referers). Non-article machinery GETs (`/robots.txt`, `/sitemap.xml`, + `/llms.txt`, the feeds) appear as + emoji-marked steps (🤖 / 🗺️ / 🧠 / 📡) so they stand out from article steps. The crawler table lists the most recent crawler first, with the most active as a tie-breaker. - **Abuse (scanner) hits**: a 404 on a telltale path — an empty URL segment diff --git a/docs/backend.md b/docs/backend.md index f4aec75..5367bc4 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -10,6 +10,7 @@ Thin FastAPI assembly: lifespan (open the kanta database, load the file store, t - `files.py` — the `FileStore` and image derivative helpers, and the file routes: `/_api/files`, `/_f/`, `/_themes/`, `/_fonts/`, the favicon settings endpoints. - `api.py` — the editor REST API and WebSockets: `/_api/pages`, `/_api/structure`, `/_api/settings`, `/_api/toggle-task`, `/_api/translations`, `/_api/ws/editor`, and the translator channel `/_translate/{clientkey}`. - `tracking.py` — visit analytics: GeoIP, client enrichment, favicon fetching, debounced broadcasts, the `/_ws` activity socket, the admin stream `/_api/ws/analytics`, and the `/_a` viewer page. +- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), all carrying every published article with full content (relative URLs absolutized), linked from every page's `` and from the sitemap, and recorded in analytics like page GETs. - `pages.py` — the public content pages: `/`, `/sitemap.xml`, `/robots.txt` and the `/{path:path}` catch-all. Route ordering is load-bearing and lives in `app.py`: the api/tracking/files routers are included BEFORE `frontend.route(app, "/")` is called — fastapi-vue inserts its file routes at the position where `route()` was called (during `load()` in the lifespan), so anything registered earlier wins. The content catch-all `/{path:path}` is included AFTER `frontend.route()` so that built frontend assets still take priority over content slugs. The `Frontend` is constructed with `spa=False` explicitly: it only serves the built files without a catch-all. diff --git a/frontend/src/analytics/format.js b/frontend/src/analytics/format.js index aa26065..09020e2 100644 --- a/frontend/src/analytics/format.js +++ b/frontend/src/analytics/format.js @@ -167,10 +167,12 @@ function externalOrigin(url) { const MACHINE_STEPS = { '/robots.txt': ['🤖', 'robots.txt'], '/sitemap.xml': ['🗺️', 'sitemap.xml'], + '/llms.txt': ['🧠', 'llms.txt'], + '/feed.json': ['📡', 'feed.json'], + '/feed.xml': ['📡', 'feed.xml'], '/rss.xml': ['📡', 'rss.xml'], '/atom.xml': ['📡', 'atom.xml'], '/feed': ['📡', 'feed'], - '/feed.xml': ['📡', 'feed.xml'], } /** Format one trail step: an internal page or an external https origin. */ diff --git a/pagerite/app.py b/pagerite/app.py index f4d39e9..5858268 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -11,6 +11,8 @@ The routes live in specialized modules, included below as APIRouters: (``/_api/*``, ``/_translate/{clientkey}``). - ``pagerite.tracking`` — visit analytics (``/_ws``, ``/_api/ws/analytics``, the ``/_a`` viewer page). +- ``pagerite.feeds`` — machine-readable site exports: ``/llms.txt``, + ``/feed.json`` (JSON Feed) and ``/feed.xml`` (RSS). - ``pagerite.pages`` — the public content pages: ``/``, ``/sitemap.xml``, ``/robots.txt`` and the ``/{path:path}`` catch-all. @@ -39,7 +41,7 @@ from fastapi.responses import Response from fastapi_vue import Frontend, env from starlette.types import ASGIApp, Receive, Scope, Send -from pagerite import api, files, pages, tracking +from pagerite import api, feeds, files, pages, tracking from pagerite.files import file_store from pagerite.state import analytics_store, config, kanta @@ -128,4 +130,5 @@ frontend.route(app, "/") # The content catch-all goes last: built assets win over content slugs, # anything unmatched falls through to content (and 404). +app.include_router(feeds.router) app.include_router(pages.router) diff --git a/pagerite/feeds.py b/pagerite/feeds.py new file mode 100644 index 0000000..b9ec363 --- /dev/null +++ b/pagerite/feeds.py @@ -0,0 +1,151 @@ +"""Machine-readable site exports: /llms.txt, /feed.json and /feed.xml. + +- ``/llms.txt`` (llmstxt.org convention): a Markdown map of the site for + LLM agents — the brand as title, then every published article as a link + with a short excerpt. +- ``/feed.json``: JSON Feed 1.1 of all published articles, full content. +- ``/feed.xml``: the same as RSS 2.0 (with an atom:link self reference) + for older feed readers. + +All three are linked from every page's (see views._layout) and from +the sitemap, recorded in analytics like page GETs (they surface as crawler +hits — no activity message ever follows them), and rendered in the site's +primary language only (feeds have no per-language negotiation here). +""" + +import json +from datetime import UTC +from email.utils import format_datetime +from xml.sax.saxutils import escape as xml_escape + +from fastapi import APIRouter, Request +from fastapi.responses import Response + +from pagerite.data import Node, node_markdown, sorted_nodes +from pagerite.markdown import make_md +from pagerite.state import SITE_URL, data +from pagerite.tracking import _record_get +from pagerite.views import _description + +router = APIRouter() + +_md = make_md() + + +def _articles() -> list[tuple[str, Node]]: + """All published content pages in menu order: (path, node).""" + out: list[tuple[str, Node]] = [] + + def walk(nodes: dict[str, Node], prefix: str) -> None: + for slug, node in sorted_nodes(nodes): + path = f"{prefix}/{slug}" if prefix else slug + if node.published and node.chunks is not None: + out.append((path, node)) + if node.children: + walk(node.children, path) + + walk(data.menu, "") + return out + + +def _body_html(node: Node, base: str) -> str: + """Full article HTML for feed content, with relative URLs absolutized. + + Rendered without the layout segmentation of page rendering (colseg + wrappers are meaningless in a feed reader); the item title carries the + page title, so no implicit h1 is injected either. + """ + html = _md.render(node_markdown(data, node) or "") + return html.replace('src="/', f'src="{base}/').replace('href="/', f'href="{base}/') + + +def _rfc822(node: Node) -> str: + return format_datetime(node.modified.astimezone(UTC), usegmt=True) + + +def _iso(node: Node) -> str: + return node.modified.astimezone(UTC).replace(microsecond=0).isoformat() + + +@router.get("/llms.txt") +async def llms_txt(request: Request) -> Response: + """Markdown map of the site for LLM agents (llmstxt.org).""" + base = SITE_URL or str(request.base_url).rstrip("/") + lines = [f"# {data.brand}", "", "## Pages", ""] + for path, node in _articles(): + url = f"{base}/{path}" if path else base + excerpt = _description(_body_html(node, base), 120) + suffix = f": {excerpt}" if excerpt else "" + lines.append(f"- [{node.title or path}]({url}){suffix}") + _record_get(request) + return Response( + "\n".join(lines) + "\n", + media_type="text/plain", + headers={"cache-control": "no-cache"}, + ) + + +@router.get("/feed.json") +async def feed_json(request: Request) -> Response: + """JSON Feed 1.1 of all published articles, full content.""" + base = SITE_URL or str(request.base_url).rstrip("/") + items = [ + { + "id": (url := f"{base}/{path}" if path else base), + "url": url, + "title": node.title or path, + "content_html": _body_html(node, base), + "date_published": node.created.astimezone(UTC) + .replace(microsecond=0) + .isoformat(), + "date_modified": _iso(node), + } + for path, node in _articles() + ] + feed = { + "version": "https://jsonfeed.org/version/1.1", + "title": data.brand, + "home_page_url": base, + "feed_url": f"{base}/feed.json", + "items": items, + } + _record_get(request) + return Response( + json.dumps(feed, ensure_ascii=False, indent=1), + media_type="application/feed+json", + headers={"cache-control": "no-cache"}, + ) + + +@router.get("/feed.xml") +async def feed_xml(request: Request) -> Response: + """RSS 2.0 of all published articles (full content in CDATA), with an + atom:link self reference.""" + base = SITE_URL or str(request.base_url).rstrip("/") + items = [] + for path, node in _articles(): + url = f"{base}/{path}" if path else base + items.append( + f"{xml_escape(node.title or path)}" + f"{xml_escape(url)}" + f'{xml_escape(url)}' + f"{_rfc822(node)}" + f"" + f"" + ) + xml = ( + '' + '' + f"{xml_escape(data.brand)}" + f"{xml_escape(base)}" + f"{xml_escape(data.brand)}" + f'' + + "".join(items) + + "" + ) + _record_get(request) + return Response( + xml, + media_type="application/rss+xml", + headers={"cache-control": "no-cache"}, + ) diff --git a/pagerite/pages.py b/pagerite/pages.py index b677f06..46e9b59 100644 --- a/pagerite/pages.py +++ b/pagerite/pages.py @@ -55,7 +55,8 @@ async def front_page(request: Request) -> Response: @router.get("/sitemap.xml") async def sitemap(request: Request) -> Response: - """Dynamically generate a sitemap of all published article pages.""" + """Dynamically generate a sitemap of all published article pages, plus + the machine-readable exports (feeds, llms.txt).""" base = SITE_URL or str(request.base_url).rstrip("/") entries: list[tuple[str, datetime, int]] = [] @@ -112,6 +113,23 @@ async def sitemap(request: Request) -> Response: ) lines.append("") + # The machine-readable exports (feeds, llms.txt) are linked too, with + # the latest article's modification time as their lastmod. + if entries: + latest = max(m for _, m, _ in entries) + lastmod = ( + latest.astimezone(UTC) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + for special in ("llms.txt", "feed.json", "feed.xml"): + lines.insert( + -1, + f" {xml_escape(f'{base}/{special}')}" + f"{lastmod}", + ) + # Recorded like a page GET: never followed by an activity message, so # it lands in the crawler list at display time (docs/analytics.md). _record_get(request) diff --git a/pagerite/views.py b/pagerite/views.py index 138c55a..0b4fb01 100644 --- a/pagerite/views.py +++ b/pagerite/views.py @@ -379,6 +379,11 @@ def _layout( doc.link(rel="canonical", href=canonical) for hreflang, href in alternates: doc.link(rel="alternate", hreflang=hreflang, href=href) + # Feed/LLM discovery links (see pagerite/feeds.py): identical on every + # page, so the positional sync (swapdoc.js) is unaffected. + doc.link(rel="alternate", type="application/feed+json", title="JSON Feed", href="/feed.json") + doc.link(rel="alternate", type="application/rss+xml", title="RSS feed", href="/feed.xml") + doc.link(rel="llms-txt", type="text/markdown", title="llms.txt", href="/llms.txt") for key, value in (social or {}).items(): if value: if key.startswith(("og:", "article:")):