Initial commit

scripts/README.md

@@ -0,0 +1,30 @@
+# Benchmark scripts
+
+Recreate every statistic and table quoted in the top-level README.
+`uarite` must be importable (e.g. `pip install -e .` or run with
+`PYTHONPATH` pointing at the repo root); the reference parsers are
+benchmark-only dependencies, pulled ad hoc via `uv run --with`.
+
+- `download_data.py` — fetch the external datasets into `data/`:
+  the 100 modern browser UAs (top-user-agents npm package) and the
+  crawler corpus (monperrus/crawler-user-agents). `data/ua.txt` is a
+  committed real-traffic sample, not downloaded.
+
+  ```
+  uv run scripts/download_data.py
+  ```
+
+- `prettytable.py` — prints the accuracy-comparison markdown table:
+
+  ```
+  uv run --with ua-parser --with user-agents python scripts/prettytable.py
+  ```
+
+- `bench.py` — prints browser-accuracy counts, crawler-detection rates,
+  URL-extraction coverage, and the timing tables (unique UAs, realistic
+  mix, bot storm) with cache statistics:
+
+  ```
+  uv run --with ua-parser --with user-agents --with user-agent-parser \
+      python scripts/bench.py
+  ```

scripts/bench.py

@@ -0,0 +1,277 @@
+"""Benchmark uarite vs ua-parser vs user-agents vs user-agent-parser.
+
+Reproduces the README's numbers: browser accuracy on 100 modern UAs,
+crawler detection on 2163 real-world crawler UAs, and timing (unique UAs,
+a realistic repeat/unique mix, a pure bot storm) with cache introspection.
+
+Data lives in scripts/data (see download_data.py).  Requires uarite
+(installed) plus the benchmark-only reference parsers:
+
+    uv run --with ua-parser --with user-agents --with user-agent-parser \
+        python scripts/bench.py
+"""
+
+import json
+import random
+import re
+import timeit
+from pathlib import Path
+
+from ua_parser import parse as ua_parse
+from user_agent_parser import parse as uap_parse
+from user_agents import parse as uas_parse
+
+from uarite import uaparse
+from uarite.core import _parse_client
+
+DATA = Path(__file__).parent / "data"
+BROWSERS = json.loads((DATA / "top-user-agents.json").read_text())
+CRAWLERS = json.loads((DATA / "crawler-user-agents.json").read_text())
+OWN = (DATA / "ua.txt").read_text().splitlines()
+CRAWLER_UAS = [ua for c in CRAWLERS for ua in (c.get("instances") or [c["pattern"]])]
+
+
+def expected(ua):
+    """De-facto ground truth for real-browser UAs: (family, major, os)."""
+    fam = major = ""
+    for tok, name in (
+        ("EdgA", "Edge"),
+        ("Edg", "Edge"),
+        ("OPR", "Opera"),
+        ("SamsungBrowser", "Samsung Internet"),
+        ("Firefox", "Firefox"),
+        ("Chrome", "Chrome"),
+    ):
+        m = re.search(re.escape(tok) + r"/(\d+)", ua)
+        if m:
+            fam, major = name, m.group(1)
+            break
+    if not fam and "Safari/" in ua:
+        m = re.search(r"Version/(\d+)", ua)
+        fam, major = "Safari", m.group(1) if m else ""
+    if not fam and ("iPhone" in ua or "iPad" in ua):
+        fam = "Safari"  # iOS webview UA: no browser token, Safari engine
+    if "iPhone" in ua or "iPad" in ua:
+        os = "ios"
+    elif re.search(r"Android [\d.]", ua):
+        os = "android"
+    elif "Windows NT" in ua:
+        os = "windows"
+    elif "Mac OS X" in ua:
+        os = "macos"
+    elif "Linux" in ua or "X11" in ua:
+        os = "linux"
+    else:
+        os = ""
+    return fam, major, os
+
+
+def norm_os(s):
+    s = (s or "").lower().replace(" ", "").replace("_", "")
+    return {"macosx": "macos", "ubuntu": "linux"}.get(s, s)
+
+
+def score_browsers():
+    res = {}
+    for name, fn in (
+        ("ua-parser", ua_parse),
+        ("user-agents", uas_parse),
+        ("user-agent-parser", uap_parse),
+    ):
+        fam_ok = ver_ok = os_ok = 0
+        for ua in BROWSERS:
+            efam, emaj, eos = expected(ua)
+            r = fn(ua)
+            if name == "ua-parser":
+                fam = r.user_agent.family or ""
+                maj = r.user_agent.major or ""
+                osf = r.os.family or ""
+            elif name == "user-agent-parser":
+                fam = r[0] or ""
+                maj = (r[1] or "").split(".")[0]
+                osf = r[2] or ""
+            else:
+                fam = r.browser.family or ""
+                maj = str(r.browser.version[0]) if r.browser.version else ""
+                osf = r.os.family or ""
+            fam = fam.split()[0]
+            fam_ok += efam.split()[0].lower() == fam.lower()
+            ver_ok += emaj == maj
+            os_ok += eos == norm_os(osf)
+        res[name] = (fam_ok, ver_ok, os_ok)
+    fam_ok = ver_ok = os_ok = 0
+    for ua in BROWSERS:
+        efam, emaj, eos = expected(ua)
+        r = uaparse(ua)
+        if emaj:
+            fam_ok += f"{efam}/{emaj}" in r.pretty
+            ver_ok += f"/{emaj}" in r.pretty
+        else:
+            # iOS webview: no version; "iPhone iOS 18" is the right answer
+            fam_ok += efam in r.pretty or "iPhone" in r.pretty or "iPad" in r.pretty
+            ver_ok += 1
+        oslabel = {
+            "ios": ("iPhone", "iPad"),
+            "macos": ("macOS",),
+            "windows": ("Windows",),
+            "linux": ("Linux",),
+            "android": ("Android",),
+        }.get(eos, ())
+        # Android with a known model drops the OS by design
+        model = re.search(r"Android [\d.]+; ([^;()]+?)(?:;|\))", ua)
+        has_model = eos == "android" and model and model.group(1) not in ("K", "Mobile")
+        os_ok += bool(any(os in r.pretty for os in oslabel) or has_model)
+    res["uarite"] = (fam_ok, ver_ok, os_ok)
+    return res
+
+
+def score_crawlers():
+    out = {}
+    crashes = 0
+    for name in ("ua-parser", "user-agents", "user-agent-parser", "uarite"):
+        det = 0
+        for ua in CRAWLER_UAS:
+            try:
+                if name == "ua-parser":
+                    r = ua_parse(ua)
+                    fam = r.user_agent.family if r.user_agent else ""
+                    bot = (r.device.family == "Spider" if r.device else False) or bool(
+                        re.search(r"bot|spider|crawl", fam, re.I)
+                    )
+                elif name == "user-agents":
+                    bot = uas_parse(ua).is_bot
+                elif name == "user-agent-parser":
+                    bot = uap_parse(ua)[4] == "Bot"
+                else:
+                    bot = bool(uaparse(ua).bot)
+            except Exception:
+                crashes += name == "user-agent-parser"
+                continue
+            det += bot
+        out[name] = det
+    url_have = sum(1 for ua in CRAWLER_UAS if re.search(r"https?://", ua))
+    url_got = sum(1 for ua in CRAWLER_UAS if uaparse(ua).url)
+    return out, url_have, url_got, crashes
+
+
+def realistic_mix():
+    """Realistic traffic: browsers repeat (weighted — popular combos many
+    times), interleaved with a steady stream of mostly-unique bot UAs that
+    hammer every parser's cache."""
+    rng = random.Random(42)
+    browsers = []
+    for i in range(2000):
+        # zipf-ish: early (popular) UAs repeat heavily
+        browsers.append(BROWSERS[int(100 * (rng.random() ** 3))])
+    bots = list(CRAWLER_UAS)
+    rng.shuffle(bots)
+    bots = bots[:1000]
+    # synthetic cache-busters: unique disguised bot UAs
+    for i in range(1000):
+        bots.append(
+            f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+            f" (KHTML, like Gecko) Chrome/{100 + i % 50}.0.{i}.0 Safari/537.36;"
+            f" compatible; ScrapeBot{i}/1.{i}; +http://scrapebot{i}.example.com/"
+        )
+    mix = [None] * (len(browsers) + len(bots))
+    mix[::2] = browsers
+    mix[1::2] = bots
+    return mix
+
+
+def bench_realistic():
+    mix = realistic_mix()
+    storm = [f"UniqueBot{i}/2.{i} (+http://ub{i}.example.com/)" for i in range(5000)]
+    print(
+        f"\n## realistic mix ({len(mix)} UAs: 2000 repeating browsers interleaved"
+        f" with 2000 mostly-unique bots)"
+    )
+    for name, fn in (
+        ("ua-parser", ua_parse),
+        ("user-agents", uas_parse),
+        ("user-agent-parser", uap_parse),
+        ("uarite", uaparse),
+    ):
+        fn = safe(fn)
+        t = timeit.timeit(lambda: [fn(u) for u in mix], number=1)
+        info = cache_info(name)
+        print(f"{name:20} {t / len(mix) * 1e6:7.1f} µs/UA   cache: {info}")
+    print(f"\n## pure bot storm ({len(storm)} unique UAs, zero cache value)")
+    for name, fn in (
+        ("ua-parser", ua_parse),
+        ("user-agents", uas_parse),
+        ("user-agent-parser", uap_parse),
+        ("uarite", uaparse),
+    ):
+        fn = safe(fn)
+        t = timeit.timeit(lambda: [fn(u) for u in storm], number=1)
+        print(
+            f"{name:20} {t / len(storm) * 1e6:7.1f} µs/UA   cache: {cache_info(name)}"
+        )
+
+
+def safe(fn):
+    def f(u):
+        try:
+            return fn(u)
+        except Exception:
+            return None
+
+    return f
+
+
+def cache_info(name):
+    if name == "uarite":
+        i = _parse_client.cache_info()
+        return f"{i.hits} hits / {i.misses} misses (cap 1024, browsers only)"
+    if name == "user-agent-parser":
+        from user_agent_parser.parser import _cached_parse_user_agent
+
+        i = _cached_parse_user_agent.cache_info()
+        return f"{i.hits} hits / {i.misses} misses (cap 512 LRU)"
+    if name == "user-agents":
+        from ua_parser.user_agent_parser import _PARSE_CACHE
+
+        return f"{len(_PARSE_CACHE)} entries (cap 200, CLEARS when full)"
+    if name == "ua-parser":
+        return "cap 2000 S3-FIFO (scan-resistant)"
+    return ""
+
+
+def bench():
+    alluas = BROWSERS + CRAWLER_UAS + OWN
+    n = 3
+    res = {}
+    for name, fn in (
+        ("ua-parser", ua_parse),
+        ("user-agents", uas_parse),
+        ("user-agent-parser", uap_parse),
+        ("uarite", uaparse),
+    ):
+        fn = safe(fn)
+        t = timeit.timeit(lambda: [fn(u) for u in alluas], number=n)
+        res[name] = t / n / len(alluas) * 1e6
+    # warm cache: repeat a small realistic working set many times
+    working = (BROWSERS + OWN[:50]) * 10
+    t = timeit.timeit(lambda: [uaparse(u) for u in working], number=n)
+    res["uarite (warm cache)"] = t / n / len(working) * 1e6
+    return res
+
+
+if __name__ == "__main__":
+    print(f"## browser accuracy (n={len(BROWSERS)}): family / version / OS correct")
+    for k, (f, v, o) in score_browsers().items():
+        print(f"{k:14} {f:3}/100   {v:3}/100   {o:3}/100")
+    det, url_have, url_got, crashes = score_crawlers()
+    print(f"\n## crawler detection (n={len(CRAWLER_UAS)})")
+    for k, v in det.items():
+        print(f"{k:18} {v:5}  ({v / len(CRAWLER_UAS):.1%})")
+    print(f"user-agent-parser crashed on {crashes} UAs")
+    print(f"\nuarite URL extraction: {url_got}/{url_have} of instances carrying a URL")
+    print(
+        "\n## speed (µs per parse, mixed set of %d UAs)"
+        % (len(BROWSERS) + len(CRAWLER_UAS) + len(OWN))
+    )
+    for k, v in bench().items():
+        print(f"{k:20} {v:8.1f}")
+    bench_realistic()

scripts/data/top-user-agents.json

@@ -0,0 +1,102 @@
+[
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36 Edg/151.0.0.0",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/126 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:154.0) Gecko/20100101 Firefox/154.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 OPR/134.0.0.0",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 OPR/134.0.0.0",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5.2 Safari/605.1.15",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5.2 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/30.0 Chrome/143.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6.1 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 Edg/152.0.0.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:154.0) Gecko/20100101 Firefox/154.0",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 26_6_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/151.0.7922.112 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Safari/605.1.15",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6.2 Safari/605.1.15",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.5 Safari/605.1.15",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.34 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.13.4 Chrome/150.0.7871.114 Electron/43.1.1 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7.5 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64; rv:154.0) Gecko/20100101 Firefox/154.0",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/27.0 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36 Edg/151.0.0.0",
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 16; SM-S928B Build/BP4A.251205.006; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/151.0.7922.199 Mobile Safari/537.36 DibburitAPK/1.6",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.3 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.2 Safari/605.1.15",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 26_5_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/151.0.7922.112 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.4 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.2 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 26_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/23G71 Twitter for iPhone/12.19.1",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 Edg/150.0.0.0",
+  "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/27.0 Safari/605.1.15",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 26_6_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/152.0.7977.64 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Mobile Safari/537.36 EdgA/151.0.0.0",
+  "Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.60 Safari/537.36 Edge/12.246",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ScalboostBrowser/4.6.0 Chrome/144.0.7559.177 Electron/40.6.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.7444.175 Safari/537.36",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 26_3_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/144.0.7559.95 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (Android 16; Mobile; rv:154.0) Gecko/154.0 Firefox/154.0"
+]

scripts/data/ua.txt

@@ -0,0 +1,316 @@
+ALittle Client
+Go-http-client/1.1
+Go-http-client/2.0
+Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.6677.221 Mobile Safari/537.36; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.7874.237 Mobile Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.5636.120 Mobile Safari/537.36; compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.5904.94 Mobile Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.6999.179 Mobile Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Extended/1.0; +http://www.google.com/bot.html) Chrome/136.0.3622.183 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.1714.57 Mobile Safari/537.36; compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.2331.116 Mobile Safari/537.36; compatible; Twitterbot/1.0
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.3871.45 Mobile Safari/537.36; compatible; Bytespider; +https://zhanzhang.toutiao.com/
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.6885.65 Mobile Safari/537.36; compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.179.134 Mobile Safari/537.36; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.304.223 Mobile Safari/537.36; compatible; GPTBot/1.4; +https://openai.com/gptbot
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.2633.116 Mobile Safari/537.36; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.3245.171 Mobile Safari/537.36; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.675.120 Mobile Safari/537.36; compatible; ClaudeBot/1.0; +claudebot@anthropic.com
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user) Chrome/136.0.5265.172 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user) Chrome/109.0.142.149 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; WhatsApp/10.0.2.1) Chrome/147.0.8581.235 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.8358.226 Mobile Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.2863.23 Mobile Safari/537.36; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.581.205 Mobile Safari/537.36; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.7679.116 Mobile Safari/537.36; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.4412.199 Mobile Safari/537.36; compatible; TelegramBot/1.0
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.1359.77 Mobile Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.3533.159 Mobile Safari/537.36; compatible; Twitterbot/1.0
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.1665.247 Mobile Safari/537.36; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/148.0.5629.104 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 15; SM-G930P; Build/AP4A.190211.226) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.1587.89 Mobile Safari/537.36 EdgA/110.0.1587.89
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.6752.19 Mobile Safari/537.36; compatible; Twitterbot/1.0
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.456.90 Mobile Safari/537.36; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.562.77 Mobile Safari/537.36; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Applebot/0.1; +http://www.apple.com/go/applebot) Chrome/130.0.4812.112 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user) Chrome/147.0.2792.237 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com) Chrome/130.0.6971.196 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot) Chrome/130.0.6187.117 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot) Chrome/147.0.8778.3 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 15; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GrokBot/1.0; +https://x.ai/grokbot) Chrome/128.0.7214.102 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)
+Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.173 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.12) Gecko/20100101 Firefox/121.12; compatible; GPTBot/1.4; +https://openai.com/gptbot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.15) Gecko/20100101 Firefox/127.15; compatible; Bytespider; +https://zhanzhang.toutiao.com/
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.7) Gecko/20100101 Firefox/133.7; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:149.4) Gecko/20100101 Firefox/149.4; compatible; ClaudeBot/1.0; +claudebot@anthropic.com
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.6) Gecko/20100101 Firefox/78.6; compatible; LinkedInBot/1.0; +http://www.linkedin.com
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.1781.181 Safari/537.36; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.7456.176 Safari/537.36 Edg/114.0.7456.176; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.4134.99 Safari/537.36; compatible; Applebot/0.1; +http://www.apple.com/go/applebot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6701.62 Safari/537.36; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.7297.17 Safari/537.36; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 CloudflareWebScanner/1.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.4253.212 Safari/537.36; compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.393.116 Safari/537.36; compatible; GPTBot/1.2; +https://openai.com/gptbot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot) Chrome/131.0.5818.195 Safari/537.36 Edg/131.0.5818.195
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Applebot/0.1; +http://www.apple.com/go/applebot) Chrome/128.0.2374.13 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bytespider; +https://zhanzhang.toutiao.com/) Chrome/122.0.8582.72 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Extended/1.0; +http://www.google.com/bot.html) Chrome/122.0.6224.91 Safari/537.36 Edg/122.0.6224.91
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.4; robots.txt; +https://openai.com/searchbot) Chrome/85.0.796.31 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot) Chrome/147.0.9065.204 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko; compatible; TelegramBot/1.0) Chrome/148.0.2316.113 Safari/537.36 Edg/148.0.2316.113
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_1; rv:119.0) Gecko/20100101 Firefox/119.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15; compatible; Bytespider; +https://zhanzhang.toutiao.com/
+Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_7) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot) Version/19.1 Safari/605.1.15
+Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_7) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; LinkedInBot/1.0; +http://www.linkedin.com) Version/18.1 Safari/605.1.15
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:127.8) Gecko/20100101 Firefox/127.8; compatible; TelegramBot/1.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:132.11) Gecko/20100101 Firefox/132.11; compatible; Twitterbot/1.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:133.5) Gecko/20100101 Firefox/133.5; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:149.5) Gecko/20100101 Firefox/149.5; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:78.16) Gecko/20100101 Firefox/78.16; compatible; ChatGPT-User/1.0; +https://openai.com/bot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.9070.130 Safari/537.36 Edg/128.0.9070.130; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.610.178 Safari/537.36; compatible; WhatsApp/10.0.2.1
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot) Chrome/151.0.779.246 Safari/537.36 Edg/151.0.779.246
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko; compatible; TelegramBot/1.0) Chrome/133.0.153.92 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko; compatible; WhatsApp/10.0.2.1) Chrome/131.0.7140.164 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler) Chrome/118.0.8850.224 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.3295.40 Safari/537.36; compatible; GPTBot/1.4; +https://openai.com/gptbot
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.1535.25 Safari/537.36; compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.6540.70 Safari/537.36; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/126.0.8218.208 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user) Chrome/128.0.3723.179 Safari/537.36 Edg/128.0.3723.179
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Extended/1.0; +http://www.google.com/bot.html) Chrome/133.0.4354.13 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko; compatible; LinkedInBot/1.0; +http://www.linkedin.com) Chrome/131.0.7882.202 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user) Chrome/131.0.2380.151 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot) Chrome/148.0.6827.81 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user
+Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user) Version/18.3 Safari/605.1.15
+Mozilla/5.0 (Macintosh; Intel Mac OS X 15_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.7 Safari/605.1.15; compatible; Applebot/0.1; +http://www.apple.com/go/applebot
+Mozilla/5.0 (Phone; OpenHarmony 6.1; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36  ArkWeb/6.1.0.120 Mobile HuaweiBrowser/6.1.6.310
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.6691.81 Safari/537.36; compatible; TelegramBot/1.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.4841.30 Safari/537.36; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.7168.194 Safari/537.36; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.2240.231 Safari/537.36; compatible; Applebot/0.1; +http://www.apple.com/go/applebot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.7895.118 Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.1055.174 Safari/537.36 Edg/133.0.1055.174; compatible; Twitterbot/1.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.2238.87 Safari/537.36 Edg/133.0.2238.87; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.3329.205 Safari/537.36; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.34 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.1090.96 Safari/537.36; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.7074.69 Safari/537.36; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user) Chrome/114.0.3989.223 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com) Chrome/151.0.8801.61 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot) Chrome/124.0.6057.104 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot) Chrome/130.0.4938.98 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots) Chrome/134.0.4631.6 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; TelegramBot/1.0) Chrome/114.0.1358.24 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; WhatsApp/10.0.2.1) Chrome/133.0.3192.3 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.5) Gecko/20100101 Firefox/105.5; compatible; ChatGPT-User/1.0; +https://openai.com/bot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0; compatible; GPTBot/1.2; +https://openai.com/gptbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.14) Gecko/20100101 Firefox/121.14; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.13) Gecko/20100101 Firefox/127.13; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.5) Gecko/20100101 Firefox/127.5; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.2) Gecko/20100101 Firefox/132.2; compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.1) Gecko/20100101 Firefox/133.1; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.14) Gecko/20100101 Firefox/133.14; compatible; ClaudeBot/1.0; +claudebot@anthropic.com
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:143.13) Gecko/20100101 Firefox/143.13; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.11) Gecko/20100101 Firefox/149.11; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:154.0) Gecko/20100101 Firefox/154.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.4) Gecko/20100101 Firefox/78.4; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler
+Mozilla/5.0 (Windows NT 10.0; rv:105.8) Gecko/20100101 Firefox/105.8; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user
+Mozilla/5.0 (Windows NT 10.0; rv:121.4) Gecko/20100101 Firefox/121.4; compatible; Twitterbot/1.0
+Mozilla/5.0 (Windows NT 10.0; rv:132.17) Gecko/20100101 Firefox/132.17; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot
+Mozilla/5.0 (Windows NT 10.0; rv:143.9) Gecko/20100101 Firefox/143.9; compatible; Applebot/0.1; +http://www.apple.com/go/applebot
+Mozilla/5.0 (Windows NT 10.0; rv:149.2) Gecko/20100101 Firefox/149.2; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user
+Mozilla/5.0 (Windows NT 10.0; rv:150.11) Gecko/20100101 Firefox/150.11; compatible; Applebot/0.1; +http://www.apple.com/go/applebot
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.2208.117 Safari/537.36; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.4903.6 Safari/537.36; compatible; ChatGPT-User/1.0; +https://openai.com/bot
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.2677.151 Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot) Chrome/109.0.1953.144 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bytespider; +https://zhanzhang.toutiao.com/) Chrome/134.0.7068.89 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot) Chrome/133.0.8680.93 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user) Chrome/124.0.1422.145 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com) Chrome/122.0.8976.2 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com) Chrome/151.0.7263.231 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot) Chrome/136.0.6556.236 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GrokBot/1.0; +https://x.ai/grokbot) Chrome/91.0.2098.213 Safari/537.36
+Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; WhatsApp/10.0.2.1) Chrome/91.0.9073.141 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.8349.207 Safari/537.36; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.870.128 Safari/537.36; compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.2718.190 Safari/537.36 Edg/114.0.2718.190; compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.673.111 Safari/537.36; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.8103.201 Safari/537.36 Edg/122.0.8103.201; compatible; GPTBot/1.4; +https://openai.com/gptbot
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6793.243 Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6864.83 Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.671.204 Safari/537.36; compatible; LinkedInBot/1.0; +http://www.linkedin.com
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.2787.208 Safari/537.36; compatible; GPTBot/1.4; +https://openai.com/gptbot
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.763.59 Safari/537.36; compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/133.0.7999.49 Safari/537.36 Edg/133.0.7999.49
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bytespider; +https://zhanzhang.toutiao.com/) Chrome/124.0.3933.247 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot) Chrome/109.0.3436.159 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot) Chrome/134.0.8299.205 Safari/537.36 Edg/134.0.8299.205
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com) Chrome/120.0.164.234 Safari/537.36 Edg/120.0.164.234
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user) Chrome/131.0.4832.111 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Extended/1.0; +http://www.google.com/bot.html) Chrome/122.0.2110.32 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GrokBot/1.0; +https://x.ai/grokbot) Chrome/148.0.7521.199 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.4; robots.txt; +https://openai.com/searchbot) Chrome/114.0.80.121 Safari/537.36
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots) Chrome/118.0.7057.134 Safari/537.36 Edg/118.0.7057.134
+Mozilla/5.0 (X11; Linux x86_64; rv:121.16) Gecko/20100101 Firefox/121.16; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots
+Mozilla/5.0 (X11; Linux x86_64; rv:121.9) Gecko/20100101 Firefox/121.9; compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 (X11; Linux x86_64; rv:127.8) Gecko/20100101 Firefox/127.8; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (X11; Linux x86_64; rv:133.16) Gecko/20100101 Firefox/133.16; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (X11; Linux x86_64; rv:149.5) Gecko/20100101 Firefox/149.5; compatible; ChatGPT-User/1.0; +https://openai.com/bot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.8500.64 Safari/537.36; compatible; Bytespider; +https://zhanzhang.toutiao.com/
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.8365.158 Safari/537.36 Edg/134.0.8365.158; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot) Chrome/128.0.4931.11 Safari/537.36
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot) Chrome/147.0.8835.247 Safari/537.36
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bytespider; +https://zhanzhang.toutiao.com/) Chrome/136.0.1808.113 Safari/537.36
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot) Chrome/136.0.3467.118 Safari/537.36
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; GrokBot/1.0; +https://x.ai/grokbot) Chrome/131.0.7648.87 Safari/537.36
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user) Chrome/133.0.1362.30 Safari/537.36
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; WhatsApp/10.0.2.1) Chrome/120.0.4740.223 Safari/537.36 Edg/120.0.4740.223
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:105.13) Gecko/20100101 Firefox/105.13; compatible; GPTBot/1.4; +https://openai.com/gptbot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:121.5) Gecko/20100101 Firefox/121.5; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:127.11) Gecko/20100101 Firefox/127.11; compatible; OAI-SearchBot/1.4; robots.txt; +https://openai.com/searchbot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:127.17) Gecko/20100101 Firefox/127.17; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:132.7) Gecko/20100101 Firefox/132.7; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:133.5) Gecko/20100101 Firefox/133.5; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:149.0) Gecko/20100101 Firefox/149.0; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:149.6) Gecko/20100101 Firefox/149.6; compatible; LinkedInBot/1.0; +http://www.linkedin.com
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:149.7) Gecko/20100101 Firefox/149.7; compatible; GPTBot/1.2; +https://openai.com/gptbot
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:150.1) Gecko/20100101 Firefox/150.1; compatible; LinkedInBot/1.0; +http://www.linkedin.com
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.8) Gecko/20100101 Firefox/78.8; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)
+Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)
+Mozilla/5.0 (compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot)
+Mozilla/5.0 (compatible; Applebot/0.1; +http://www.apple.com/go/applebot)
+Mozilla/5.0 (compatible; Bytespider; +https://zhanzhang.toutiao.com/)
+Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)
+Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot)
+Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)
+Mozilla/5.0 (compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user)
+Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)
+Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)
+Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)
+Mozilla/5.0 (compatible; GPTBot/1.4; +https://openai.com/gptbot)
+Mozilla/5.0 (compatible; Google-Extended/1.0; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Mozilla/5.0 (compatible; GrokBot/1.0; +https://x.ai/grokbot)
+Mozilla/5.0 (compatible; LinkedInBot/1.0; +http://www.linkedin.com)
+Mozilla/5.0 (compatible; MJ12bot/v2.0.5; http://mj12bot.com/)
+Mozilla/5.0 (compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot)
+Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)
+Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)
+Mozilla/5.0 (compatible; Reflectionbot/1.0; +https://reflection.ai/bot)
+Mozilla/5.0 (compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots)
+Mozilla/5.0 (compatible; TelegramBot/1.0)
+Mozilla/5.0 (compatible; Twitterbot/1.0)
+Mozilla/5.0 (compatible; WP-Safe-Scanner/1.0)
+Mozilla/5.0 (compatible; WhatsApp/10.0.2.1)
+Mozilla/5.0 (compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php)
+Mozilla/5.0 (compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler)
+Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3 Mobile/15E148 Safari/604.1; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (iPhone; CPU iPhone OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/19.7 Mobile/15E148 Safari/604.1; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (iPhone; CPU iPhone OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot) Version/19.1 Mobile/15E148 Safari/604.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 16_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; Google-Extended/1.0; +http://www.google.com/bot.html) Version/16.6 Mobile/15E148 Safari/604.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.1229.179 Mobile Safari/537.36; compatible; Bytespider; +https://zhanzhang.toutiao.com/
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.903.10 Mobile Safari/537.36; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.4431.64 Mobile Safari/537.36; compatible; GPTBot/1.2; +https://openai.com/gptbot
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.4849.17 Mobile Safari/537.36; compatible; TelegramBot/1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.672.19 Mobile Safari/537.36; compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.790.154 Mobile Safari/537.36; compatible; LinkedInBot/1.0; +http://www.linkedin.com
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com) Chrome/151.0.5552.152 Mobile Safari/537.36
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Discordbot/2.0; +https://discordapp.com) Chrome/131.0.8856.25 Mobile Safari/537.36
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot) Version/16.5 Mobile/15E148 Safari/604.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; TelegramBot/1.0) Version/18.6 Mobile/15E148 Safari/604.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Mobile/15E148 Safari/604.1; compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/19.0 Mobile/15E148 Safari/604.1; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.8733.214 Mobile Safari/537.36; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.7561.171 Mobile Safari/537.36; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bytespider; +https://zhanzhang.toutiao.com/) Chrome/128.0.6985.201 Mobile Safari/537.36
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.4; robots.txt; +https://openai.com/searchbot) Chrome/131.0.3919.233 Mobile Safari/537.36
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user) Chrome/118.0.5358.73 Mobile Safari/537.36
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; compatible; meta-externalagent/1.1; +https://developers.facebook.com/docs/sharing/webmasters/crawler) Chrome/122.0.3316.88 Mobile Safari/537.36
+Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1; compatible; TelegramBot/1.0
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Applebot/0.1; +http://www.apple.com/go/applebot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Claude-User/1.0; +Claude-User@anthropic.com
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ClaudeBot/1.0; +claudebot@anthropic.com
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Discordbot/2.0; +https://discordapp.com
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Google-Extended/1.0; +http://www.google.com/bot.html
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GrokBot/1.0; +https://x.ai/grokbot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; LinkedInBot/1.0; +http://www.linkedin.com
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; robots.txt; +https://openai.com/searchbot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; TelegramBot/1.0
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Twitterbot/1.0
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; WhatsApp/10.0.2.1
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/1.0; +https://developer.amazon.com/support/amazonbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Applebot/0.1; +http://www.apple.com/go/applebot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bytespider; +https://zhanzhang.toutiao.com/)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +https://www.anthropic.com/claude-user)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Discordbot/2.0; +https://discordapp.com)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Extended/1.0; +http://www.google.com/bot.html)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GrokBot/1.0; +https://x.ai/grokbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; LinkedInBot/1.0; +http://www.linkedin.com)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.3; +https://openai.com/searchbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.4; robots.txt; +https://openai.com/searchbot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Slackbot-LinkExpanding/1.0; +https://api.slack.com/robots)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; TelegramBot/1.0)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Twitterbot/1.0)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; WhatsApp/10.0.2.1)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/136.0.0.0 Safari/537.36
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/136.0.0.0 Safari/537.36 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php)
+crusader-worker/1.0
+null
+pip/24.3.1 {"ci":null,"cpu":"x86_64","implementation":{"name":"CPython","version":"3.11.9"},"installer":{"name":"pip","version":"24.3.1"},"openssl_version":"OpenSSL 3.0.13 30 Jan 2024","python":"3.11.9","setuptools_version":"75.6.0","system":{"name":"Linux","release":"6.8.0"}}
+python-httpx/0.28.1
+python-requests/2.25.1
+python-requests/2.32.5
+sibsweb-verifier/0.1 (+mailto:ptwebcrawler@techremotehub.com)

scripts/download_data.py

@@ -0,0 +1,52 @@
+"""Download the external datasets used by the benchmark scripts.
+
+- data/top-user-agents.json — 100 current real-browser UAs, from the
+  top-user-agents npm package (src/index.json inside the tarball).
+- data/crawler-user-agents.json — crawler patterns with sample UAs, from
+  github.com/monperrus/crawler-user-agents.
+
+data/ua.txt is not downloaded: it is a real traffic sample (unique
+User-Agent strings seen by one site), committed alongside this script.
+"""
+
+import json
+import tarfile
+import urllib.request
+from io import BytesIO
+from pathlib import Path
+
+DATA = Path(__file__).parent / "data"
+
+CRAWLERS_URL = (
+    "https://raw.githubusercontent.com/monperrus/crawler-user-agents/"
+    "master/crawler-user-agents.json"
+)
+NPM_REGISTRY = "https://registry.npmjs.org/top-user-agents/latest"
+
+
+def fetch(url: str) -> bytes:
+    with urllib.request.urlopen(url, timeout=120) as r:
+        return r.read()
+
+
+def main() -> None:
+    DATA.mkdir(exist_ok=True)
+
+    crawlers = DATA / "crawler-user-agents.json"
+    print(f"downloading {CRAWLERS_URL}")
+    crawlers.write_bytes(fetch(CRAWLERS_URL))
+
+    print("resolving latest top-user-agents from the npm registry")
+    meta = json.loads(fetch(NPM_REGISTRY))
+    tarball = fetch(meta["dist"]["tarball"])
+    with tarfile.open(fileobj=BytesIO(tarball), mode="r:gz") as tar:
+        top = tar.extractfile("package/src/index.json").read()
+    (DATA / "top-user-agents.json").write_bytes(top)
+
+    for f in ("crawler-user-agents.json", "top-user-agents.json"):
+        n = len(json.loads((DATA / f).read_text()))
+        print(f"{f}: {n} entries")
+
+
+if __name__ == "__main__":
+    main()

scripts/prettytable.py

@@ -0,0 +1,104 @@
+"""Print the README's accuracy-comparison table as markdown.
+
+uarite's column is simply `r.pretty`.  The reference modules have no
+display format; their columns use one plain format string each on their
+structured output (footnotes ²³ in the README).  Requires uarite (installed)
+plus the benchmark-only reference parsers:
+
+    uv run --with ua-parser --with user-agents python scripts/prettytable.py
+"""
+
+from ua_parser import parse as ua_parse
+from user_agents import parse as uas_parse
+
+from uarite import uaparse
+
+CASES = [
+    (
+        "Chrome, Windows",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
+    ),
+    (
+        "Safari, macOS",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15",
+    ),
+    (
+        "Opera, Linux",
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 OPR/106.0.0.0",
+    ),
+    (
+        "Chrome, Android (no model)",
+        "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Mobile Safari/537.36",
+    ),
+    (
+        "Chrome, Android (Pixel)",
+        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Mobile Safari/537.36",
+    ),
+    (
+        "Edge, Android (model code)",
+        "Mozilla/5.0 (Linux; Android 15; SM-G930P; Build/AP4A.190211.226) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.1587.89 Mobile Safari/537.36 EdgA/110.0.1587.89",
+    ),
+    (
+        "Firefox, Android",
+        "Mozilla/5.0 (Android 15; Mobile; rv:154.0) Gecko/154.0 Firefox/154.0",
+    ),
+    (
+        "Safari, iPhone",
+        "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
+    ),
+    (
+        "Huawei HarmonyOS phone",
+        "Mozilla/5.0 (Phone; OpenHarmony 6.1; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36  ArkWeb/6.1.0.120 Mobile HuaweiBrowser/6.1.6.310",
+    ),
+    ("GPTBot", "Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)"),
+    (
+        "Googlebot (disguised)",
+        "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    ),
+    (
+        "Claude-SearchBot (disguised)",
+        "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.3245.171 Mobile Safari/537.36; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/claude-searchbot",
+    ),
+    (
+        "Facebook preview (disguised)",
+        "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.6885.65 Mobile Safari/537.36; compatible; facebookexternalhit/1.1; +http://www.facebook.com/externalhit_uatext.php",
+    ),
+    (
+        "Meta crawler (disguised)",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))",
+    ),
+    ("WhatsApp preview", "Mozilla/5.0 (compatible; WhatsApp/10.0.2.1)"),
+    (
+        "Bytespider",
+        "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)",
+    ),
+    (
+        "BingPreview",
+        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",
+    ),
+    ("AhrefsBot", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"),
+    ("python-requests", "python-requests/2.32.5"),
+]
+
+
+def imitate_uaparser(ua):
+    r = ua_parse(ua)
+    fam = (r.user_agent.family if r.user_agent else "") or ""
+    maj = (r.user_agent.major if r.user_agent else "") or ""
+    osf = (r.os.family if r.os else "") or ""
+    dev = (r.device.family if r.device else "") or ""
+    return f"{fam}/{maj} {osf} {dev}"
+
+
+def imitate_useragents(ua):
+    r = uas_parse(ua)
+    fam = r.browser.family or ""
+    maj = str(r.browser.version[0]) if r.browser.version else ""
+    return f"{fam}/{maj} {r.os.family or ''} {r.device.family or ''}"
+
+
+print("| Case | uarite¹ | ua-parser² | user-agents³ |")
+print("|---|---|---|---|")
+for name, ua in CASES:
+    ours = uaparse(ua).pretty or "—"
+    print(f"| {name} | {ours} | {imitate_uaparser(ua)} | {imitate_useragents(ua)} |")