LeoVasanko/pagerite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Crawlers should include all sorts of spiders along with bots and googleother.

Browse Source
This commit is contained in:
LeoVasanko
2026-08-24 05:28:05 +00:00
parent 7b8899af92
commit 075848f782
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pagerite/analytics.py
+5 -5
View File
@@ -246,15 +246,15 @@ def _utm_tags(query: str) -> dict[str, str]:
_CRAWLER_TIMEOUT = timedelta(seconds=10) _CRAWLER_TIMEOUT = timedelta(seconds=10)
#: UAs of JS-running crawlers, which would register as visitors on their #: UAs of JS-running crawlers, which would register as visitors on their
#: ping. Anything calling itself a "bot" matches; known crawlers without #: ping. Anything calling itself a "bot" or "spider" matches; known crawlers
#: that token (GoogleOther) are listed as extra alternates. No source #: without those tokens (GoogleOther) are listed as extra alternates. No
#: verification: a spoofed bot UA just lands in the crawler list, and #: source verification: a spoofed bot UA just lands in the crawler list, and
#: scanners that probe telltale paths are caught by the abuse rules anyway. #: scanners that probe telltale paths are caught by the abuse rules anyway.
_BOT_UA = re.compile(r"bot|googleother", re.IGNORECASE) _BOT_UA = re.compile(r"bot|spider|googleother", re.IGNORECASE)
def _is_bot_ua(ua: str) -> bool: def _is_bot_ua(ua: str) -> bool:
"""True when the UA claims a crawler identity (Googlebot, Applebot, ...).""" """True when the UA claims a crawler identity (bot or spider)."""
return bool(_BOT_UA.search(ua)) return bool(_BOT_UA.search(ua))
#: Plain-404 count per IP that classifies it as abuse even without a #: Plain-404 count per IP that classifies it as abuse even without a