From f1e16b7abeeda87facff729de938d6747a13d9b2 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Sat, 25 Apr 2026 00:20:19 +0000 Subject: [PATCH] Preview: add pdf+pyvips encode path and simplify timings --- cista/preview.py | 91 +++++++++++++++++++++++---------------- cista/preview_worker.py | 14 +++--- scripts/preview_images.py | 36 ++++++++-------- 3 files changed, 79 insertions(+), 62 deletions(-) diff --git a/cista/preview.py b/cista/preview.py index 592fa11..a75c662 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -21,6 +21,7 @@ import av import fitz # PyMuPDF import numpy as np import pillow_heif +import pyvips from blake3 import blake3 from PIL import Image from sanic import Blueprint, empty, raw, redirect @@ -107,9 +108,17 @@ class _PreviewWorker: if self.proc.stdin is None or self.proc.stdout is None: raise WorkerProtocolError("worker streams not available") - line = msgspec.json.encode( - PreviewRequest(path=str(filepath), quality=quality, maxsize=maxsize, maxzoom=maxzoom) - ) + b"\n" + line = ( + msgspec.json.encode( + PreviewRequest( + path=str(filepath), + quality=quality, + maxsize=maxsize, + maxzoom=maxzoom, + ) + ) + + b"\n" + ) self.proc.stdin.write(line) await self.proc.stdin.drain() @@ -211,7 +220,9 @@ class _PreviewWorkerPool: msgspec.json.DecodeError, ) as e: replace = True - logger.warning("Preview worker protocol failure for %s: %s", filepath.name, e) + logger.warning( + "Preview worker protocol failure for %s: %s", filepath.name, e + ) raise PreviewError(filepath.name) finally: if replace: @@ -231,7 +242,9 @@ class _PreviewWorkerPool: self._idle.get_nowait() except asyncio.QueueEmpty: break - await asyncio.gather(*(worker.kill() for worker in workers), return_exceptions=True) + await asyncio.gather( + *(worker.kill() for worker in workers), return_exceptions=True + ) async def start_preview_workers() -> None: @@ -263,7 +276,9 @@ async def shutdown_preview_workers() -> None: proc.kill() except ProcessLookupError: pass - await asyncio.gather(*(proc.wait() for proc in list(_active_procs)), return_exceptions=True) + await asyncio.gather( + *(proc.wait() for proc in list(_active_procs)), return_exceptions=True + ) _active_procs.clear() @@ -348,20 +363,21 @@ async def preview(req, path): # Generate preview try: - img, preview_resp = await _run_preview_process(filepath, quality, maxsize, maxzoom) + img, preview_resp = await _run_preview_process( + filepath, quality, maxsize, maxzoom + ) except PreviewTimeout: return empty(504) except PreviewError: return empty(422) if preview_resp and preview_resp.backend: - if preview_resp.load_ms is not None: - load_ms = int(round(preview_resp.load_ms)) - process_ms = int(round(preview_resp.process_ms or 0.0)) - save_ms = int(round(preview_resp.save_ms or 0.0)) - timing_detail = f"{load_ms}/{process_ms}/{save_ms}" + if preview_resp.timings: + timing_detail = "/".join( + str(int(round(value))) for value in preview_resp.timings + ) + req.ctx._log_extra = f"{preview_resp.backend} {timing_detail} ➛" else: - timing_detail = str(int(round(preview_resp.total_ms or 0.0))) - req.ctx._log_extra = f"{preview_resp.backend} {timing_detail} ➛" + req.ctx._log_extra = preview_resp.backend if not img: # Preview generation failed, redirect to the file itself return redirect(f"/files/{path}", status=303) @@ -409,16 +425,10 @@ def process_image(path, *, maxsize, quality): def process_image_with_timing(path, *, maxsize, quality): if FORCE_PIL: return process_image_pillow(path, maxsize=maxsize, quality=quality) - try: - return process_image_pyvips(path, maxsize=maxsize, quality=quality) - except Exception as e: - logger.debug("Falling back to Pillow preview for %s: %s", path.name, e) - return process_image_pillow(path, maxsize=maxsize, quality=quality) + return process_image_pyvips(path, maxsize=maxsize, quality=quality) def process_image_pyvips(path, *, maxsize, quality): - import pyvips - t_start = perf_counter() img = pyvips.Image.new_from_file(str(path), access="sequential") img = img.autorot() @@ -437,7 +447,7 @@ def process_image_pyvips(path, *, maxsize, quality): ok=True, mime="image/avif", backend="pyvips", - total_ms=round((t_end - t_start) * 1000, 1), + timings=[round((t_end - t_start) * 1000, 1)], ) @@ -476,10 +486,7 @@ def process_image_pillow(path, *, maxsize, quality): ok=True, mime="image/avif", backend="pillow", - load_ms=round(load_ms, 1), - process_ms=round(proc_ms, 1), - save_ms=round(save_ms, 1), - total_ms=round((t_end - t_load) * 1000, 1), + timings=[round(load_ms, 1), round(proc_ms, 1), round(save_ms, 1)], ) @@ -494,18 +501,29 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0): t_load_end = perf_counter() t_save_start = perf_counter() - ret = pix.pil_tobytes( - format="avif", quality=quality, speed=10, max_threads=1, avif=1 - ) + if FORCE_PIL: + ret = pix.pil_tobytes( + format="avif", quality=quality, speed=10, max_threads=1, avif=1 + ) + backend = "pdf" + else: + img = pyvips.Image.new_from_memory( + pix.samples_mv, pix.width, pix.height, pix.n, "uchar" + ) + ret = img.write_to_buffer( + ".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True + ) + backend = "pdf+pyvips" t_save_end = perf_counter() return ret, PreviewResponse( ok=True, mime="image/avif", - backend="pdf", - load_ms=round((t_load_end - t_load_start) * 1000, 1), - save_ms=round((t_save_end - t_save_start) * 1000, 1), - total_ms=round((t_save_end - t_load_start) * 1000, 1), + backend=backend, + timings=[ + round((t_load_end - t_load_start) * 1000, 1), + round((t_save_end - t_save_start) * 1000, 1), + ], ) @@ -623,9 +641,10 @@ def process_video(path, *, maxsize, quality): ok=True, mime="image/avif", backend="video", - load_ms=round((t_load_end - t_load_start) * 1000, 1), - save_ms=round((t_save_end - t_save_start) * 1000, 1), - total_ms=round((t_save_end - t_load_start) * 1000, 1), + timings=[ + round((t_load_end - t_load_start) * 1000, 1), + round((t_save_end - t_save_start) * 1000, 1), + ], ) del imgdata, istream, ostream, icc, occ, frame gc.collect() diff --git a/cista/preview_worker.py b/cista/preview_worker.py index 945a432..791f2a4 100644 --- a/cista/preview_worker.py +++ b/cista/preview_worker.py @@ -8,6 +8,7 @@ Framed response format: (blake3(packet))(uint32 json size)(uint32 payload size)(json)(binary payload) where packet = (uint32 json size)(uint32 payload size)(json)(binary payload). """ + import logging import struct import sys @@ -28,10 +29,7 @@ class PreviewResponse(msgspec.Struct, omit_defaults=True): ok: bool mime: str | None = None backend: str | None = None - load_ms: float | None = None - process_ms: float | None = None - save_ms: float | None = None - total_ms: float | None = None + timings: list[float] | None = None error: str | None = None @@ -50,9 +48,7 @@ def _write_response(resp: PreviewResponse, payload: bytes) -> None: def _run_once() -> None: if len(sys.argv) != 5: - sys.stderr.write( - f"Usage: {sys.argv[0]} \n" - ) + sys.stderr.write(f"Usage: {sys.argv[0]} \n") sys.exit(1) from cista.preview import dispatch @@ -76,7 +72,9 @@ def _run_loop() -> None: return try: req = _dec_req.decode(line) - result, resp = dispatch(Path(req.path), req.quality, req.maxsize, req.maxzoom) + result, resp = dispatch( + Path(req.path), req.quality, req.maxsize, req.maxzoom + ) _write_response(resp, result or b"") except Exception as e: _write_response(PreviewResponse(ok=False, error=str(e)), b"") diff --git a/scripts/preview_images.py b/scripts/preview_images.py index 2cec854..b99249b 100644 --- a/scripts/preview_images.py +++ b/scripts/preview_images.py @@ -36,16 +36,16 @@ def main() -> int: if not folder.is_dir(): raise SystemExit(f"Not a directory: {folder}") - files = sorted(path for path in folder.rglob("*") if path.is_file() and is_image_file(path)) + files = sorted( + path for path in folder.rglob("*") if path.is_file() and is_image_file(path) + ) if not files: print(f"No image files found under {folder}") return 0 total_files = 0 total_bytes = 0 - total_load_ms: float = 0.0 - total_process_ms: float = 0.0 - total_save_ms: float = 0.0 + total_timing_slots: list[float] = [] total_preview_ms: float = 0.0 failures = 0 @@ -67,18 +67,19 @@ def main() -> int: continue total_bytes += len(preview) - total_load_ms += timing.load_ms or 0.0 - total_process_ms += timing.process_ms or 0.0 - total_save_ms += timing.save_ms or 0.0 - total_preview_ms += timing.total_ms or 0.0 + timings = timing.timings or [] + if len(total_timing_slots) < len(timings): + total_timing_slots.extend([0.0] * (len(timings) - len(total_timing_slots))) + for i, value in enumerate(timings): + total_timing_slots[i] += value + total_ms = sum(timings) + total_preview_ms += total_ms - if timing.load_ms is not None: - detail = ( - f"load={timing.load_ms:.1f}ms process={timing.process_ms:.1f}ms " - f"save={timing.save_ms:.1f}ms total={timing.total_ms:.1f}ms" - ) + detail = " / ".join(f"{value:.1f}ms" for value in timings) + if detail: + detail = f"timings={detail} total={total_ms:.1f}ms" else: - detail = f"total={timing.total_ms:.1f}ms" + detail = f"total={total_ms:.1f}ms" print(f"OK {rel} backend={timing.backend} bytes={len(preview)} {detail}") completed = total_files - failures @@ -89,10 +90,9 @@ def main() -> int: print(f" failed={failures}") print(f" preview_bytes={total_bytes}") if completed: - if total_load_ms or total_process_ms or total_save_ms: - print(f" load_total_ms={total_load_ms:.1f}") - print(f" process_total_ms={total_process_ms:.1f}") - print(f" save_total_ms={total_save_ms:.1f}") + if total_timing_slots: + for i, value in enumerate(total_timing_slots, start=1): + print(f" timing{i}_total_ms={value:.1f}") print(f" preview_total_ms={total_preview_ms:.1f}") print(f" preview_avg_ms={total_preview_ms / completed:.1f}") return 0 if failures == 0 else 1