diff --git a/docs/analytics.md b/docs/analytics.md index 90f3d05..b37780a 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -123,7 +123,8 @@ WebSocket reports actual navigations and active time spent on a page. message handling is never delayed. The decompressed `dbip-*.mmdb` file is kept in the repository root and ignored by git. The CLI flag `--dbip` (`uv run pagerite --dbip`) downloads the latest - `dbip-city-lite-YYYY-MM.mmdb.gz` from DB-IP before the server starts, + `dbip-city-lite-YYYY-MM.mmdb.gz` from DB-IP at startup (in the app + lifespan, before the MMDB is opened), skipping the download when the local database is already current and removing older versions after an update; without the flag only an existing file is used. diff --git a/pagerite/__main__.py b/pagerite/__main__.py index 932dd4f..7d49941 100644 --- a/pagerite/__main__.py +++ b/pagerite/__main__.py @@ -1,80 +1,15 @@ """Command-line entry point for running the backend server.""" import argparse -import gzip import os -import sys -from datetime import date from pathlib import Path -import httpx from fastapi_vue import server from fastapi_vue.hostutil import parse_endpoints DEFAULT_PORT = 8100 DEVMODE = os.getenv("PAGERITE_DEV") == "1" -# Repository root (pagerite/__main__.py -> ..), where the MMDB lives. -_REPO_ROOT = Path(__file__).resolve().parent.parent - -DBIP_URL = "https://download.db-ip.com/free/dbip-city-lite-{month}.mmdb.gz" - - -def _download_dbip() -> None: - """Download the latest dbip-city-lite MMDB if ours is missing or older.""" - today = date.today() - months = [f"{today:%Y-%m}"] - # The current month's file may not be published yet; fall back to last month. - prev = (today.replace(day=1) - date.resolution).replace(day=1) - months.append(f"{prev:%Y-%m}") - - existing = sorted( - p.stem.removeprefix("dbip-city-lite-").removesuffix(".mmdb") - for p in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*") - ) - if existing and existing[-1] >= months[0]: - print( - f"pagerite: DB-IP database is current ({existing[-1]}), skipping download" - ) - return - - for month in months: - url = DBIP_URL.format(month=month) - target = _REPO_ROOT / f"dbip-city-lite-{month}.mmdb.gz" - tmp = target.with_suffix(".mmdb.gz.tmp") - print(f"pagerite: downloading {url}") - try: - with httpx.stream("GET", url, follow_redirects=True, timeout=120) as r: - if r.status_code == 404: - continue - r.raise_for_status() - with open(tmp, "wb") as f: - for chunk in r.iter_bytes(): - f.write(chunk) - except httpx.HTTPError as e: - print(f"pagerite: DB-IP download failed: {e}", file=sys.stderr) - tmp.unlink(missing_ok=True) - continue - # Verify it is actually gzip data before installing it. - try: - with gzip.open(tmp, "rb") as f: - f.read(1) - except OSError: - print( - f"pagerite: DB-IP download for {month} was not valid gzip", - file=sys.stderr, - ) - tmp.unlink(missing_ok=True) - continue - os.replace(tmp, target) - # Drop older databases so the app never picks up a stale one. - for old in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*"): - if old.name != target.name: - old.unlink() - print(f"pagerite: DB-IP database updated to {target.name}") - return - print("pagerite: could not download a DB-IP database", file=sys.stderr) - def main() -> None: """Run the backend server with optional arguments.""" @@ -110,8 +45,10 @@ def main() -> None: if "port" in endpoint: os.environ["PAGERITE_PORT"] = str(endpoint["port"]) break + # --dbip: the app lifespan downloads/updates the DB-IP database, where + # logging is already set up. if args.dbip: - _download_dbip() + os.environ["PAGERITE_DBIP"] = "1" server.run( "pagerite.app:app", listen=args.listen, diff --git a/pagerite/app.py b/pagerite/app.py index 5cca3fa..36be04d 100644 --- a/pagerite/app.py +++ b/pagerite/app.py @@ -30,6 +30,7 @@ walking the tree (``resolve``), moves are slot detach/attach import asyncio import logging +import os from contextlib import asynccontextmanager from fastapi import FastAPI, Request @@ -49,8 +50,11 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator: async with kanta: await asyncio.to_thread(file_store.load) await frontend.load() - # Decompress/open the DB-IP MMDB once at startup. Lookups are then - # read-only and safe to run in background ``to_thread`` workers. + # --dbip: update the DB-IP database first, then decompress/open the + # MMDB once. Lookups are then read-only and safe to run in + # background ``to_thread`` workers. + if os.environ.get("PAGERITE_DBIP") == "1": + await asyncio.to_thread(tracking._download_dbip) await asyncio.to_thread(tracking._geoip._load) analytics_store.subscribe(tracking._schedule_analytics_broadcast) # Backfill favicons for external sites already in the recorded data. diff --git a/pagerite/tracking.py b/pagerite/tracking.py index 2b16430..2d4a1c3 100644 --- a/pagerite/tracking.py +++ b/pagerite/tracking.py @@ -16,6 +16,7 @@ import os import re import shutil import socket +from datetime import date from functools import lru_cache from pathlib import Path from urllib.parse import urlparse @@ -45,6 +46,59 @@ _analytics_broadcast_task: asyncio.Task | None = None # Repository root from this file's location (pagerite/tracking.py -> ..). _REPO_ROOT = Path(__file__).resolve().parent.parent +DBIP_URL = "https://download.db-ip.com/free/dbip-city-lite-{month}.mmdb.gz" + + +def _download_dbip() -> None: + """Download the latest dbip-city-lite MMDB if ours is missing or older.""" + today = date.today() + months = [f"{today:%Y-%m}"] + # The current month's file may not be published yet; fall back to last month. + prev = (today.replace(day=1) - date.resolution).replace(day=1) + months.append(f"{prev:%Y-%m}") + + existing = sorted( + p.stem.removeprefix("dbip-city-lite-").removesuffix(".mmdb") + for p in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*") + ) + if existing and existing[-1] >= months[0]: + logger.info("DB-IP database is current (%s), skipping download", existing[-1]) + return + + for month in months: + url = DBIP_URL.format(month=month) + target = _REPO_ROOT / f"dbip-city-lite-{month}.mmdb.gz" + tmp = target.with_suffix(".mmdb.gz.tmp") + logger.info("Downloading %s", url) + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=120) as r: + if r.status_code == 404: + continue + r.raise_for_status() + with open(tmp, "wb") as f: + for chunk in r.iter_bytes(): + f.write(chunk) + except httpx.HTTPError as e: + logger.warning("DB-IP download failed: %s", e) + tmp.unlink(missing_ok=True) + continue + # Verify it is actually gzip data before installing it. + try: + with gzip.open(tmp, "rb") as f: + f.read(1) + except OSError: + logger.warning("DB-IP download for %s was not valid gzip", month) + tmp.unlink(missing_ok=True) + continue + os.replace(tmp, target) + # Drop older databases so the app never picks up a stale one. + for old in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*"): + if old.name != target.name: + old.unlink() + logger.info("DB-IP database updated to %s", target.name) + return + logger.warning("Could not download a DB-IP database") + def _geoip_db_path() -> Path | None: """Find a DB-IP MMDB in the repo root, preferring an already-decompressed