Normalize bot pretty names; drop undocumented FacebookBot
PRETTY_OVERRIDE in bots.py replaces name-plus-label composition with full per-bot pretty strings. The messy X-Google / Google-X names print as "Google X (kind)"; Googlebot keeps its canonical name. FacebookBot is no longer in Meta's crawler documentation, so it falls back to generic spider detection; Facebook rejoins provider Meta but prints bare, like the other social unfurlers.
This commit is contained in:
+15
-2
@@ -40,7 +40,6 @@ BOTS = {
|
|||||||
"meta-externalagent": ("Meta-ExternalAgent", "ai"),
|
"meta-externalagent": ("Meta-ExternalAgent", "ai"),
|
||||||
"meta-externalfetcher": ("Meta-ExternalFetcher", "ai"),
|
"meta-externalfetcher": ("Meta-ExternalFetcher", "ai"),
|
||||||
"meta-webindexer": ("Meta-WebIndexer", "search"),
|
"meta-webindexer": ("Meta-WebIndexer", "search"),
|
||||||
"facebookbot": ("FacebookBot", "ai"),
|
|
||||||
"bingpreview": ("BingPreview", "search"),
|
"bingpreview": ("BingPreview", "search"),
|
||||||
"pinterest": ("Pinterest", "social"),
|
"pinterest": ("Pinterest", "social"),
|
||||||
"embedly": ("Embedly", "social"),
|
"embedly": ("Embedly", "social"),
|
||||||
@@ -86,7 +85,6 @@ PROVIDERS = {
|
|||||||
"Microsoft": frozenset({"Bingbot", "BingPreview"}),
|
"Microsoft": frozenset({"Bingbot", "BingPreview"}),
|
||||||
"Meta": frozenset({
|
"Meta": frozenset({
|
||||||
"Facebook",
|
"Facebook",
|
||||||
"FacebookBot",
|
|
||||||
"Meta-ExternalAgent",
|
"Meta-ExternalAgent",
|
||||||
"Meta-ExternalFetcher",
|
"Meta-ExternalFetcher",
|
||||||
"Meta-WebIndexer",
|
"Meta-WebIndexer",
|
||||||
@@ -98,6 +96,21 @@ PROVIDER_OF = {
|
|||||||
name: provider for provider, names in PROVIDERS.items() for name in names
|
name: provider for provider, names in PROVIDERS.items() for name in names
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#: Per-bot pretty overrides: the full display string, replacing the
|
||||||
|
#: name-plus-kind-label composition entirely.
|
||||||
|
PRETTY_OVERRIDE = {
|
||||||
|
"Facebook": "Facebook",
|
||||||
|
"Feedfetcher-Google": "Google Feedfetcher (search)",
|
||||||
|
"Google-InspectionTool": "Google InspectionTool (search)",
|
||||||
|
"Google-Read-Aloud": "Google Read-Aloud (AI)",
|
||||||
|
"Mediapartners-Google": "Google Mediapartners (analytics)",
|
||||||
|
"AdsBot-Google": "Google AdsBot (analytics)",
|
||||||
|
"APIs-Google": "Google APIs",
|
||||||
|
"Storebot-Google": "Google Storebot (search)",
|
||||||
|
"Google-Extended": "Google Extended (AI)",
|
||||||
|
"GoogleOther": "Google Other (AI)",
|
||||||
|
}
|
||||||
|
|
||||||
#: Reverse lookup: bot display name -> kind.
|
#: Reverse lookup: bot display name -> kind.
|
||||||
NAME_KIND = {name: kind for name, kind in BOTS.values()}
|
NAME_KIND = {name: kind for name, kind in BOTS.values()}
|
||||||
|
|
||||||
|
|||||||
+5
-3
@@ -4,7 +4,7 @@ import re
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
|
||||||
from .bots import BOTS, KIND_LABEL, LABELED, PROVIDER_OF
|
from .bots import BOTS, KIND_LABEL, LABELED, PRETTY_OVERRIDE, PROVIDER_OF
|
||||||
from .clients import BROWSERS, SAMSUNG, SAMSUNG_SERIES
|
from .clients import BROWSERS, SAMSUNG, SAMSUNG_SERIES
|
||||||
|
|
||||||
|
|
||||||
@@ -169,8 +169,10 @@ def uaparse(ua: str | None) -> UA:
|
|||||||
if name:
|
if name:
|
||||||
# The browser/OS in crawler UAs is a disguise; the bot identity is
|
# The browser/OS in crawler UAs is a disguise; the bot identity is
|
||||||
# the relevant information, so ``engine`` and ``os`` are left empty.
|
# the relevant information, so ``engine`` and ``os`` are left empty.
|
||||||
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
|
pretty = PRETTY_OVERRIDE.get(name)
|
||||||
pretty = f"{name} ({label})" if label else name
|
if pretty is None:
|
||||||
|
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
|
||||||
|
pretty = f"{name} ({label})" if label else name
|
||||||
return UA(
|
return UA(
|
||||||
pretty=pretty, kind=kind, url=url(ua),
|
pretty=pretty, kind=kind, url=url(ua),
|
||||||
provider=PROVIDER_OF.get(name, ""),
|
provider=PROVIDER_OF.get(name, ""),
|
||||||
|
|||||||
Reference in New Issue
Block a user