Normalize bot pretty names; drop undocumented FacebookBot

PRETTY_OVERRIDE in bots.py replaces name-plus-label composition with full per-bot pretty strings. The messy X-Google / Google-X names print as "Google X (kind)"; Googlebot keeps its canonical name. FacebookBot is no longer in Meta's crawler documentation, so it falls back to generic spider detection; Facebook rejoins provider Meta but prints bare, like the other social unfurlers.
This commit is contained in:
2026-09-08 23:36:31 +00:00
parent dd59fcb832
commit 887d492598
2 changed files with 20 additions and 5 deletions
+15 -2
View File
@@ -40,7 +40,6 @@ BOTS = {
"meta-externalagent": ("Meta-ExternalAgent", "ai"),
"meta-externalfetcher": ("Meta-ExternalFetcher", "ai"),
"meta-webindexer": ("Meta-WebIndexer", "search"),
"facebookbot": ("FacebookBot", "ai"),
"bingpreview": ("BingPreview", "search"),
"pinterest": ("Pinterest", "social"),
"embedly": ("Embedly", "social"),
@@ -86,7 +85,6 @@ PROVIDERS = {
"Microsoft": frozenset({"Bingbot", "BingPreview"}),
"Meta": frozenset({
"Facebook",
"FacebookBot",
"Meta-ExternalAgent",
"Meta-ExternalFetcher",
"Meta-WebIndexer",
@@ -98,6 +96,21 @@ PROVIDER_OF = {
name: provider for provider, names in PROVIDERS.items() for name in names
}
#: Per-bot pretty overrides: the full display string, replacing the
#: name-plus-kind-label composition entirely.
PRETTY_OVERRIDE = {
"Facebook": "Facebook",
"Feedfetcher-Google": "Google Feedfetcher (search)",
"Google-InspectionTool": "Google InspectionTool (search)",
"Google-Read-Aloud": "Google Read-Aloud (AI)",
"Mediapartners-Google": "Google Mediapartners (analytics)",
"AdsBot-Google": "Google AdsBot (analytics)",
"APIs-Google": "Google APIs",
"Storebot-Google": "Google Storebot (search)",
"Google-Extended": "Google Extended (AI)",
"GoogleOther": "Google Other (AI)",
}
#: Reverse lookup: bot display name -> kind.
NAME_KIND = {name: kind for name, kind in BOTS.values()}
+5 -3
View File
@@ -4,7 +4,7 @@ import re
from dataclasses import dataclass
from functools import lru_cache
from .bots import BOTS, KIND_LABEL, LABELED, PROVIDER_OF
from .bots import BOTS, KIND_LABEL, LABELED, PRETTY_OVERRIDE, PROVIDER_OF
from .clients import BROWSERS, SAMSUNG, SAMSUNG_SERIES
@@ -169,8 +169,10 @@ def uaparse(ua: str | None) -> UA:
if name:
# The browser/OS in crawler UAs is a disguise; the bot identity is
# the relevant information, so ``engine`` and ``os`` are left empty.
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
pretty = f"{name} ({label})" if label else name
pretty = PRETTY_OVERRIDE.get(name)
if pretty is None:
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
pretty = f"{name} ({label})" if label else name
return UA(
pretty=pretty, kind=kind, url=url(ua),
provider=PROVIDER_OF.get(name, ""),