Add engine and os fields to UA, restructure README prose, restyle pyproject
This commit is contained in:
@@ -1,31 +1,32 @@
|
||||
# uarite
|
||||
# User-Agent parsed Right
|
||||
|
||||
User-Agent parsing in Python has a long lineage. ua-parser is the official Python implementation of the ua-parser project, built around uap-core: the regex database extracted from BrowserScope's original parser and shared by implementations in many languages. user-agents wraps ua-parser with higher-level device and capability detection; its last release was in 2020. user-agent-parser is a separate implementation first released in 2022 and substantially updated in 2026, taking its own approach rather than building on uap-core. None of the three has further dependencies, but the regex databases weigh something: ua-parser installs at about 499 KB (531 KB with user-agents on top), user-agent-parser at 166 KB.
|
||||
User-Agent parsing in Python has a long lineage. ua-parser is the official Python implementation of the ua-parser project, built around uap-core: the regex database extracted from BrowserScope's original parser and shared by implementations in many languages. user-agents wraps ua-parser with higher-level device and capability detection but its last release was in 2020. user-agent-parser is a separate implementation first released in 2022 and substantially updated in 2026, taking its own approach rather than building on uap-core. None of the three has further dependencies, but the regex databases weigh something: ua-parser and user-agents each install about half a megabyte, user-agent-parser at 166 kB. We are merely 29 kB and yet perform better especially with the new crawlers of the AI boom.
|
||||
|
||||
This module is another take on the same problem: a small, dependency-free, compact pure-Python parser. It returns structured classifications, but also the thing most applications eventually need: **a short human-readable pretty description**.
|
||||
|
||||
Add to your project:
|
||||
|
||||
```sh
|
||||
uv add uarite
|
||||
```
|
||||
|
||||
This module is another take on the same problem: a small, dependency-free, compact pure-Python parser — 29 KB installed. It returns structured classifications, but also the thing most applications eventually need: a short human-readable description.
|
||||
|
||||
It is particularly aimed at server-side analytics, where correctly recognizing crawlers and modern reduced User-Agents matters. It detects disguised crawlers, distinguishes AI/search/preview traffic, handles HarmonyOS and bots without calling them Android, resolves common device model codes and falls back to reasonable output even when all else fails.
|
||||
We correctly detect disguised crawlers, distinguish traffic of AI learning, search engines and social media share previews. We handle HarmonyOS and bots without calling them Android, resolving common device model codes, and fall back to reasonable output even when all else fails.
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from uarite import uaparse
|
||||
|
||||
r = uaparse("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36")
|
||||
r = uaparse("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36")
|
||||
|
||||
r.pretty # "Chrome/152 Windows"
|
||||
r.engine # "Chromium"
|
||||
r.os # "Windows"
|
||||
r.kind # "browser"
|
||||
r.bot # ""
|
||||
r.url # ""
|
||||
|
||||
r = uaparse("Mozilla/5.0 (Linux; Android 13; Pixel 7) ... Chrome/134.0.6885.65 "
|
||||
"Mobile Safari/537.36; compatible; facebookexternalhit/1.1; "
|
||||
"+http://www.facebook.com/externalhit_uatext.php")
|
||||
r = uaparse("Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.6885.65 Mobile Safari/537.36; compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php")
|
||||
|
||||
r.pretty # "Facebook"
|
||||
r.kind # "preview"
|
||||
@@ -40,10 +41,14 @@ r.url # "http://www.facebook.com/externalhit_uatext.php"
|
||||
| Field | Content |
|
||||
| -------- | ------------------------------------------------------------------------------------------------ |
|
||||
| `pretty` | Compact display string (below); `""` for empty/missing UAs, the raw UA when unrecognized |
|
||||
| `engine` | `"Chromium"`, `"Gecko"`, `"Safari"`, `"ArkWeb"` (HarmonyOS), or `""` |
|
||||
| `os` | `"Windows"`, `"macOS"`, `"Linux"`, `"iOS"`, `"Android"`, `"HarmonyOS"`, or `""` |
|
||||
| `bot` | Crawler/previewer display name, or `""` |
|
||||
| `kind` | `"browser"`, `"ai"`, `"search"`, `"preview"`, `"spider"`, or `""` (scripts/HTTP libraries) |
|
||||
| `url` | The crawler's info URL (`+https://…` pointer), or `""`; not part of `pretty` — link it in the UI |
|
||||
|
||||
`os` is the major OS only, no version — meant for things like offering OS-specific downloads. `engine` is derived from the browser identity: every recognized browser is Chromium except Firefox/LibreWolf (Gecko) and Safari and all of iOS (Safari's engine is all Apple allows there); HarmonyOS browsers run ArkWeb. Both are left empty for crawlers: the browser and OS in a disguised crawler UA are part of the disguise.
|
||||
|
||||
`kind` is `"browser"` for Mozilla-format UAs with no bot token, `"ai"` for training-data and AI-assistant fetchers (GPTBot, ClaudeBot, Google-Extended, ...), `"search"` for search-engine indexing (Googlebot, Bingbot, ...), `"preview"` for social link-preview fetchers (Facebook, WhatsApp, Slack, ...), `"spider"` for generic or unknown crawlers, and `""` for scripts and HTTP libraries.
|
||||
|
||||
`pretty` is intended to be shown directly:
|
||||
|
||||
+1
-3
@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
|
||||
[project]
|
||||
name = "uarite"
|
||||
dynamic = ["version"]
|
||||
description = "Compact, human-readable User-Agent formatting and bot detection — no regex database, stdlib only"
|
||||
description = "User-Agent parsing done right. Accurate, small, pure Python and fast."
|
||||
authors = [
|
||||
{ name = "Leo Vasanko" },
|
||||
]
|
||||
@@ -18,9 +18,7 @@ requires-python = ">=3.10"
|
||||
classifiers = [
|
||||
"Intended Audience :: Developers",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
]
|
||||
dependencies = []
|
||||
|
||||
|
||||
+25
-9
@@ -11,6 +11,8 @@ from .clients import BROWSERS, SAMSUNG, SAMSUNG_SERIES
|
||||
@dataclass(frozen=True)
|
||||
class UA:
|
||||
pretty: str
|
||||
engine: str
|
||||
os: str
|
||||
bot: str
|
||||
kind: str
|
||||
url: str
|
||||
@@ -93,6 +95,16 @@ def browser(ua: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
#: Engines that differ from the Chromium default for recognized browsers.
|
||||
ENGINES = {"Firefox": "Gecko", "LibreWolf": "Gecko", "Safari": "Safari"}
|
||||
|
||||
|
||||
def engine(b: str) -> str:
|
||||
"""Engine for a ``Browser/major`` result; Chromium is the modern default."""
|
||||
name = b.split("/")[0]
|
||||
return ENGINES.get(name, "Chromium" if name else "")
|
||||
|
||||
|
||||
def os(ua: str) -> str:
|
||||
"""Desktop OS name, or "" when not recognizable."""
|
||||
if "Windows NT" in ua:
|
||||
@@ -136,14 +148,14 @@ def uaparse(ua: str) -> UA:
|
||||
would just flush the cache.
|
||||
"""
|
||||
if not ua or not ua.strip() or ua in ("-", "null"):
|
||||
return UA("", "", "", "")
|
||||
return UA("", "", "", "", "", "")
|
||||
name, kind = bot(ua)
|
||||
if name:
|
||||
# The browser/OS in crawler UAs is a disguise; the bot identity is
|
||||
# the relevant information.
|
||||
# the relevant information, so ``engine`` and ``os`` are left empty.
|
||||
label = KIND_LABEL.get(kind, "") if name in LABELED else ""
|
||||
pretty = f"{name} ({label})" if label else name
|
||||
return UA(pretty, name, kind, url(ua))
|
||||
return UA(pretty, "", "", name, kind, url(ua))
|
||||
return _parse_client(ua)
|
||||
|
||||
|
||||
@@ -160,19 +172,22 @@ def _parse_client(ua: str) -> UA:
|
||||
os_name = os(ua)
|
||||
if os_name and os_name not in pretty:
|
||||
pretty = f"{pretty} {os_name}"
|
||||
return UA(pretty, "", "", "")
|
||||
return UA(pretty, "", os_name, "", "", "")
|
||||
|
||||
# HarmonyOS carries an "Android" compatibility token, so it must be
|
||||
# detected before Android.
|
||||
if "OpenHarmony" in ua or "HarmonyOS" in ua or "ArkWeb" in ua:
|
||||
b = browser(ua)
|
||||
return UA(f"{b} HarmonyOS" if b else "HarmonyOS", "", "browser", "")
|
||||
return UA(
|
||||
f"{b} HarmonyOS" if b else "HarmonyOS", "ArkWeb", "HarmonyOS",
|
||||
"", "browser", "",
|
||||
)
|
||||
|
||||
if "iPhone" in ua or "iPad" in ua:
|
||||
device = "iPhone" if "iPhone" in ua else "iPad"
|
||||
m = re.search(r"OS (\d+)", ua)
|
||||
pretty = f"{device} iOS {m.group(1)}" if m else device
|
||||
return UA(pretty, "", "browser", "")
|
||||
return UA(pretty, "Safari", "iOS", "", "browser", "")
|
||||
|
||||
m = re.search(r"Android ([\d.]+)", ua)
|
||||
if m:
|
||||
@@ -190,11 +205,12 @@ def _parse_client(ua: str) -> UA:
|
||||
if token and token not in ("wv", "Mobile", "Tablet"):
|
||||
model = model_name(token)
|
||||
parts = [p for p in (b, model or f"Android {m.group(1)}") if p]
|
||||
return UA(" ".join(parts), "", "browser", "")
|
||||
return UA(" ".join(parts), engine(b), "Android", "", "browser", "")
|
||||
|
||||
b = browser(ua)
|
||||
pretty = f"{b} {os(ua)}".strip()
|
||||
return UA(pretty or ua, "", "browser", "")
|
||||
os_name = os(ua)
|
||||
pretty = f"{b} {os_name}".strip()
|
||||
return UA(pretty or ua, engine(b), os_name, "", "browser", "")
|
||||
|
||||
|
||||
def is_bot(ua: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user