From 075848f78246ff45ac531f22163488959807950d Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Mon, 24 Aug 2026 05:28:05 +0000 Subject: [PATCH] Crawlers should include all sorts of spiders along with bots and googleother. --- pagerite/analytics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pagerite/analytics.py b/pagerite/analytics.py index a58a204..7dad3a9 100644 --- a/pagerite/analytics.py +++ b/pagerite/analytics.py @@ -246,15 +246,15 @@ def _utm_tags(query: str) -> dict[str, str]: _CRAWLER_TIMEOUT = timedelta(seconds=10) #: UAs of JS-running crawlers, which would register as visitors on their -#: ping. Anything calling itself a "bot" matches; known crawlers without -#: that token (GoogleOther) are listed as extra alternates. No source -#: verification: a spoofed bot UA just lands in the crawler list, and +#: ping. Anything calling itself a "bot" or "spider" matches; known crawlers +#: without those tokens (GoogleOther) are listed as extra alternates. No +#: source verification: a spoofed bot UA just lands in the crawler list, and #: scanners that probe telltale paths are caught by the abuse rules anyway. -_BOT_UA = re.compile(r"bot|googleother", re.IGNORECASE) +_BOT_UA = re.compile(r"bot|spider|googleother", re.IGNORECASE) def _is_bot_ua(ua: str) -> bool: - """True when the UA claims a crawler identity (Googlebot, Applebot, ...).""" + """True when the UA claims a crawler identity (bot or spider).""" return bool(_BOT_UA.search(ua)) #: Plain-404 count per IP that classifies it as abuse even without a