From 447a565b050620eede4b1d0c3b1f19ada09f41ae Mon Sep 17 00:00:00 2001
From: Leo Vasanko <git@vasanko.com>
Date: Sat, 29 Aug 2026 20:45:39 +0000
Subject: [PATCH] Disallow /auth/ and /_api in robots.txt

---
 pagerite/app.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pagerite/app.py b/pagerite/app.py
index e10f96f..72e3473 100644
--- a/pagerite/app.py
+++ b/pagerite/app.py
@@ -1459,9 +1459,11 @@ async def sitemap(request: Request) -> Response:
 
 @app.get("/robots.txt")
 async def robots_txt(request: Request) -> Response:
-    """Allow all crawling and point crawlers at the sitemap."""
+    """Allow content crawling, keep the SSO login (/auth/) and the
+    admin-gated API (/_api) out of search results, and point crawlers at
+    the sitemap."""
     base = SITE_URL or str(request.base_url).rstrip("/")
-    body = f"User-agent: *\nAllow: /\nSitemap: {base}/sitemap.xml\n"
+    body = f"User-agent: *\nAllow: /\nDisallow: /auth/\nDisallow: /_api\nSitemap: {base}/sitemap.xml\n"
     return Response(
         body,
         media_type="text/plain",