Initial commit

scripts/download_data.py

@@ -0,0 +1,52 @@
+"""Download the external datasets used by the benchmark scripts.
+
+- data/top-user-agents.json — 100 current real-browser UAs, from the
+  top-user-agents npm package (src/index.json inside the tarball).
+- data/crawler-user-agents.json — crawler patterns with sample UAs, from
+  github.com/monperrus/crawler-user-agents.
+
+data/ua.txt is not downloaded: it is a real traffic sample (unique
+User-Agent strings seen by one site), committed alongside this script.
+"""
+
+import json
+import tarfile
+import urllib.request
+from io import BytesIO
+from pathlib import Path
+
+DATA = Path(__file__).parent / "data"
+
+CRAWLERS_URL = (
+    "https://raw.githubusercontent.com/monperrus/crawler-user-agents/"
+    "master/crawler-user-agents.json"
+)
+NPM_REGISTRY = "https://registry.npmjs.org/top-user-agents/latest"
+
+
+def fetch(url: str) -> bytes:
+    with urllib.request.urlopen(url, timeout=120) as r:
+        return r.read()
+
+
+def main() -> None:
+    DATA.mkdir(exist_ok=True)
+
+    crawlers = DATA / "crawler-user-agents.json"
+    print(f"downloading {CRAWLERS_URL}")
+    crawlers.write_bytes(fetch(CRAWLERS_URL))
+
+    print("resolving latest top-user-agents from the npm registry")
+    meta = json.loads(fetch(NPM_REGISTRY))
+    tarball = fetch(meta["dist"]["tarball"])
+    with tarfile.open(fileobj=BytesIO(tarball), mode="r:gz") as tar:
+        top = tar.extractfile("package/src/index.json").read()
+    (DATA / "top-user-agents.json").write_bytes(top)
+
+    for f in ("crawler-user-agents.json", "top-user-agents.json"):
+        n = len(json.loads((DATA / f).read_text()))
+        print(f"{f}: {n} entries")
+
+
+if __name__ == "__main__":
+    main()