diff --git a/pagerite/app.py b/pagerite/app.py index e10f96f..72e3473 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -1459,9 +1459,11 @@ async def sitemap(request: Request) -> Response: @app.get("/robots.txt") async def robots_txt(request: Request) -> Response: - """Allow all crawling and point crawlers at the sitemap.""" + """Allow content crawling, keep the SSO login (/auth/) and the + admin-gated API (/_api) out of search results, and point crawlers at + the sitemap.""" base = SITE_URL or str(request.base_url).rstrip("/") - body = f"User-agent: *\nAllow: /\nSitemap: {base}/sitemap.xml\n" + body = f"User-agent: *\nAllow: /\nDisallow: /auth/\nDisallow: /_api\nSitemap: {base}/sitemap.xml\n" return Response( body, media_type="text/plain",