Normalize bot pretty names; drop undocumented FacebookBot
PRETTY_OVERRIDE in bots.py replaces name-plus-label composition with full per-bot pretty strings. The messy X-Google / Google-X names print as "Google X (kind)"; Googlebot keeps its canonical name. FacebookBot is no longer in Meta's crawler documentation, so it falls back to generic spider detection; Facebook rejoins provider Meta but prints bare, like the other social unfurlers.
This commit is contained in:
+15
-2
@@ -40,7 +40,6 @@ BOTS = {
|
||||
"meta-externalagent": ("Meta-ExternalAgent", "ai"),
|
||||
"meta-externalfetcher": ("Meta-ExternalFetcher", "ai"),
|
||||
"meta-webindexer": ("Meta-WebIndexer", "search"),
|
||||
"facebookbot": ("FacebookBot", "ai"),
|
||||
"bingpreview": ("BingPreview", "search"),
|
||||
"pinterest": ("Pinterest", "social"),
|
||||
"embedly": ("Embedly", "social"),
|
||||
@@ -86,7 +85,6 @@ PROVIDERS = {
|
||||
"Microsoft": frozenset({"Bingbot", "BingPreview"}),
|
||||
"Meta": frozenset({
|
||||
"Facebook",
|
||||
"FacebookBot",
|
||||
"Meta-ExternalAgent",
|
||||
"Meta-ExternalFetcher",
|
||||
"Meta-WebIndexer",
|
||||
@@ -98,6 +96,21 @@ PROVIDER_OF = {
|
||||
name: provider for provider, names in PROVIDERS.items() for name in names
|
||||
}
|
||||
|
||||
#: Per-bot pretty overrides: the full display string, replacing the
|
||||
#: name-plus-kind-label composition entirely.
|
||||
PRETTY_OVERRIDE = {
|
||||
"Facebook": "Facebook",
|
||||
"Feedfetcher-Google": "Google Feedfetcher (search)",
|
||||
"Google-InspectionTool": "Google InspectionTool (search)",
|
||||
"Google-Read-Aloud": "Google Read-Aloud (AI)",
|
||||
"Mediapartners-Google": "Google Mediapartners (analytics)",
|
||||
"AdsBot-Google": "Google AdsBot (analytics)",
|
||||
"APIs-Google": "Google APIs",
|
||||
"Storebot-Google": "Google Storebot (search)",
|
||||
"Google-Extended": "Google Extended (AI)",
|
||||
"GoogleOther": "Google Other (AI)",
|
||||
}
|
||||
|
||||
#: Reverse lookup: bot display name -> kind.
|
||||
NAME_KIND = {name: kind for name, kind in BOTS.values()}
|
||||
|
||||
|
||||
+5
-3
@@ -4,7 +4,7 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
|
||||
from .bots import BOTS, KIND_LABEL, LABELED, PROVIDER_OF
|
||||
from .bots import BOTS, KIND_LABEL, LABELED, PRETTY_OVERRIDE, PROVIDER_OF
|
||||
from .clients import BROWSERS, SAMSUNG, SAMSUNG_SERIES
|
||||
|
||||
|
||||
@@ -169,8 +169,10 @@ def uaparse(ua: str | None) -> UA:
|
||||
if name:
|
||||
# The browser/OS in crawler UAs is a disguise; the bot identity is
|
||||
# the relevant information, so ``engine`` and ``os`` are left empty.
|
||||
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
|
||||
pretty = f"{name} ({label})" if label else name
|
||||
pretty = PRETTY_OVERRIDE.get(name)
|
||||
if pretty is None:
|
||||
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
|
||||
pretty = f"{name} ({label})" if label else name
|
||||
return UA(
|
||||
pretty=pretty, kind=kind, url=url(ua),
|
||||
provider=PROVIDER_OF.get(name, ""),
|
||||
|
||||
Reference in New Issue
Block a user