Move DB-IP download into app lifespan with proper logging

docs/analytics.md

@@ -123,7 +123,8 @@ WebSocket reports actual navigations and active time spent on a page.
   message handling is never delayed.  The decompressed `dbip-*.mmdb` file is kept in
   the repository root and ignored by git.  The CLI flag `--dbip`
   (`uv run pagerite --dbip`) downloads the latest
-  `dbip-city-lite-YYYY-MM.mmdb.gz` from DB-IP before the server starts,
+  `dbip-city-lite-YYYY-MM.mmdb.gz` from DB-IP at startup (in the app
+  lifespan, before the MMDB is opened),
   skipping the download when the local database is already current and
   removing older versions after an update; without the flag only an existing
   file is used.

pagerite/__main__.py

@@ -1,80 +1,15 @@
 """Command-line entry point for running the backend server."""
 
 import argparse
-import gzip
 import os
-import sys
-from datetime import date
 from pathlib import Path
 
-import httpx
 from fastapi_vue import server
 from fastapi_vue.hostutil import parse_endpoints
 
 DEFAULT_PORT = 8100
 DEVMODE = os.getenv("PAGERITE_DEV") == "1"
 
-# Repository root (pagerite/__main__.py -> ..), where the MMDB lives.
-_REPO_ROOT = Path(__file__).resolve().parent.parent
-
-DBIP_URL = "https://download.db-ip.com/free/dbip-city-lite-{month}.mmdb.gz"
-
-
-def _download_dbip() -> None:
-    """Download the latest dbip-city-lite MMDB if ours is missing or older."""
-    today = date.today()
-    months = [f"{today:%Y-%m}"]
-    # The current month's file may not be published yet; fall back to last month.
-    prev = (today.replace(day=1) - date.resolution).replace(day=1)
-    months.append(f"{prev:%Y-%m}")
-
-    existing = sorted(
-        p.stem.removeprefix("dbip-city-lite-").removesuffix(".mmdb")
-        for p in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*")
-    )
-    if existing and existing[-1] >= months[0]:
-        print(
-            f"pagerite: DB-IP database is current ({existing[-1]}), skipping download"
-        )
-        return
-
-    for month in months:
-        url = DBIP_URL.format(month=month)
-        target = _REPO_ROOT / f"dbip-city-lite-{month}.mmdb.gz"
-        tmp = target.with_suffix(".mmdb.gz.tmp")
-        print(f"pagerite: downloading {url}")
-        try:
-            with httpx.stream("GET", url, follow_redirects=True, timeout=120) as r:
-                if r.status_code == 404:
-                    continue
-                r.raise_for_status()
-                with open(tmp, "wb") as f:
-                    for chunk in r.iter_bytes():
-                        f.write(chunk)
-        except httpx.HTTPError as e:
-            print(f"pagerite: DB-IP download failed: {e}", file=sys.stderr)
-            tmp.unlink(missing_ok=True)
-            continue
-        # Verify it is actually gzip data before installing it.
-        try:
-            with gzip.open(tmp, "rb") as f:
-                f.read(1)
-        except OSError:
-            print(
-                f"pagerite: DB-IP download for {month} was not valid gzip",
-                file=sys.stderr,
-            )
-            tmp.unlink(missing_ok=True)
-            continue
-        os.replace(tmp, target)
-        # Drop older databases so the app never picks up a stale one.
-        for old in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*"):
-            if old.name != target.name:
-                old.unlink()
-        print(f"pagerite: DB-IP database updated to {target.name}")
-        return
-    print("pagerite: could not download a DB-IP database", file=sys.stderr)
-
 
 def main() -> None:
     """Run the backend server with optional arguments."""
@@ -110,8 +45,10 @@ def main() -> None:
         if "port" in endpoint:
             os.environ["PAGERITE_PORT"] = str(endpoint["port"])
             break
+    # --dbip: the app lifespan downloads/updates the DB-IP database, where
+    # logging is already set up.
     if args.dbip:
-        _download_dbip()
+        os.environ["PAGERITE_DBIP"] = "1"
     server.run(
         "pagerite.app:app",
         listen=args.listen,

pagerite/app.py

@@ -30,6 +30,7 @@ walking the tree (``resolve``), moves are slot detach/attach
 
 import asyncio
 import logging
+import os
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, Request
@@ -49,8 +50,11 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator:
     async with kanta:
         await asyncio.to_thread(file_store.load)
         await frontend.load()
-        # Decompress/open the DB-IP MMDB once at startup.  Lookups are then
-        # read-only and safe to run in background ``to_thread`` workers.
+        # --dbip: update the DB-IP database first, then decompress/open the
+        # MMDB once.  Lookups are then read-only and safe to run in
+        # background ``to_thread`` workers.
+        if os.environ.get("PAGERITE_DBIP") == "1":
+            await asyncio.to_thread(tracking._download_dbip)
         await asyncio.to_thread(tracking._geoip._load)
         analytics_store.subscribe(tracking._schedule_analytics_broadcast)
         # Backfill favicons for external sites already in the recorded data.

pagerite/tracking.py

@@ -16,6 +16,7 @@ import os
 import re
 import shutil
 import socket
+from datetime import date
 from functools import lru_cache
 from pathlib import Path
 from urllib.parse import urlparse
@@ -45,6 +46,59 @@ _analytics_broadcast_task: asyncio.Task | None = None
 # Repository root from this file's location (pagerite/tracking.py -> ..).
 _REPO_ROOT = Path(__file__).resolve().parent.parent
 
+DBIP_URL = "https://download.db-ip.com/free/dbip-city-lite-{month}.mmdb.gz"
+
+
+def _download_dbip() -> None:
+    """Download the latest dbip-city-lite MMDB if ours is missing or older."""
+    today = date.today()
+    months = [f"{today:%Y-%m}"]
+    # The current month's file may not be published yet; fall back to last month.
+    prev = (today.replace(day=1) - date.resolution).replace(day=1)
+    months.append(f"{prev:%Y-%m}")
+
+    existing = sorted(
+        p.stem.removeprefix("dbip-city-lite-").removesuffix(".mmdb")
+        for p in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*")
+    )
+    if existing and existing[-1] >= months[0]:
+        logger.info("DB-IP database is current (%s), skipping download", existing[-1])
+        return
+
+    for month in months:
+        url = DBIP_URL.format(month=month)
+        target = _REPO_ROOT / f"dbip-city-lite-{month}.mmdb.gz"
+        tmp = target.with_suffix(".mmdb.gz.tmp")
+        logger.info("Downloading %s", url)
+        try:
+            with httpx.stream("GET", url, follow_redirects=True, timeout=120) as r:
+                if r.status_code == 404:
+                    continue
+                r.raise_for_status()
+                with open(tmp, "wb") as f:
+                    for chunk in r.iter_bytes():
+                        f.write(chunk)
+        except httpx.HTTPError as e:
+            logger.warning("DB-IP download failed: %s", e)
+            tmp.unlink(missing_ok=True)
+            continue
+        # Verify it is actually gzip data before installing it.
+        try:
+            with gzip.open(tmp, "rb") as f:
+                f.read(1)
+        except OSError:
+            logger.warning("DB-IP download for %s was not valid gzip", month)
+            tmp.unlink(missing_ok=True)
+            continue
+        os.replace(tmp, target)
+        # Drop older databases so the app never picks up a stale one.
+        for old in _REPO_ROOT.glob("dbip-city-lite-*.mmdb*"):
+            if old.name != target.name:
+                old.unlink()
+        logger.info("DB-IP database updated to %s", target.name)
+        return
+    logger.warning("Could not download a DB-IP database")
+
 
 def _geoip_db_path() -> Path | None:
     """Find a DB-IP MMDB in the repo root, preferring an already-decompressed