Vasanko/cista-storage
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preview: add pdf+pyvips encode path and simplify timings

Browse Source
This commit is contained in:
LeoVasanko
2026-04-25 00:20:19 +00:00
parent 0ebff0ec17
commit f1e16b7abe
3 changed files with 79 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cista/preview.py
+51 -32
View File
@@ -21,6 +21,7 @@ import av
import fitz # PyMuPDF import fitz # PyMuPDF
import numpy as np import numpy as np
import pillow_heif import pillow_heif
import pyvips
from blake3 import blake3 from blake3 import blake3
from PIL import Image from PIL import Image
from sanic import Blueprint, empty, raw, redirect from sanic import Blueprint, empty, raw, redirect
@@ -107,9 +108,17 @@ class _PreviewWorker:
if self.proc.stdin is None or self.proc.stdout is None: if self.proc.stdin is None or self.proc.stdout is None:
raise WorkerProtocolError("worker streams not available") raise WorkerProtocolError("worker streams not available")
line = msgspec.json.encode( line = (
PreviewRequest(path=str(filepath), quality=quality, maxsize=maxsize, maxzoom=maxzoom) msgspec.json.encode(
) + b"\n" PreviewRequest(
path=str(filepath),
quality=quality,
maxsize=maxsize,
maxzoom=maxzoom,
)
)
+ b"\n"
)
self.proc.stdin.write(line) self.proc.stdin.write(line)
await self.proc.stdin.drain() await self.proc.stdin.drain()
@@ -211,7 +220,9 @@ class _PreviewWorkerPool:
msgspec.json.DecodeError, msgspec.json.DecodeError,
) as e: ) as e:
replace = True replace = True
logger.warning("Preview worker protocol failure for %s: %s", filepath.name, e) logger.warning(
"Preview worker protocol failure for %s: %s", filepath.name, e
)
raise PreviewError(filepath.name) raise PreviewError(filepath.name)
finally: finally:
if replace: if replace:
@@ -231,7 +242,9 @@ class _PreviewWorkerPool:
self._idle.get_nowait() self._idle.get_nowait()
except asyncio.QueueEmpty: except asyncio.QueueEmpty:
break break
await asyncio.gather(*(worker.kill() for worker in workers), return_exceptions=True) await asyncio.gather(
*(worker.kill() for worker in workers), return_exceptions=True
)
async def start_preview_workers() -> None: async def start_preview_workers() -> None:
@@ -263,7 +276,9 @@ async def shutdown_preview_workers() -> None:
proc.kill() proc.kill()
except ProcessLookupError: except ProcessLookupError:
pass pass
await asyncio.gather(*(proc.wait() for proc in list(_active_procs)), return_exceptions=True) await asyncio.gather(
*(proc.wait() for proc in list(_active_procs)), return_exceptions=True
)
_active_procs.clear() _active_procs.clear()
@@ -348,20 +363,21 @@ async def preview(req, path):
# Generate preview # Generate preview
try: try:
img, preview_resp = await _run_preview_process(filepath, quality, maxsize, maxzoom) img, preview_resp = await _run_preview_process(
filepath, quality, maxsize, maxzoom
)
except PreviewTimeout: except PreviewTimeout:
return empty(504) return empty(504)
except PreviewError: except PreviewError:
return empty(422) return empty(422)
if preview_resp and preview_resp.backend: if preview_resp and preview_resp.backend:
if preview_resp.load_ms is not None: if preview_resp.timings:
load_ms = int(round(preview_resp.load_ms)) timing_detail = "/".join(
process_ms = int(round(preview_resp.process_ms or 0.0)) str(int(round(value))) for value in preview_resp.timings
save_ms = int(round(preview_resp.save_ms or 0.0)) )
timing_detail = f"{load_ms}/{process_ms}/{save_ms}"
else:
timing_detail = str(int(round(preview_resp.total_ms or 0.0)))
req.ctx._log_extra = f"{preview_resp.backend} {timing_detail} ➛" req.ctx._log_extra = f"{preview_resp.backend} {timing_detail} ➛"
else:
req.ctx._log_extra = preview_resp.backend
if not img: if not img:
# Preview generation failed, redirect to the file itself # Preview generation failed, redirect to the file itself
return redirect(f"/files/{path}", status=303) return redirect(f"/files/{path}", status=303)
@@ -409,16 +425,10 @@ def process_image(path, *, maxsize, quality):
def process_image_with_timing(path, *, maxsize, quality): def process_image_with_timing(path, *, maxsize, quality):
if FORCE_PIL: if FORCE_PIL:
return process_image_pillow(path, maxsize=maxsize, quality=quality) return process_image_pillow(path, maxsize=maxsize, quality=quality)
try:
return process_image_pyvips(path, maxsize=maxsize, quality=quality) return process_image_pyvips(path, maxsize=maxsize, quality=quality)
except Exception as e:
logger.debug("Falling back to Pillow preview for %s: %s", path.name, e)
return process_image_pillow(path, maxsize=maxsize, quality=quality)
def process_image_pyvips(path, *, maxsize, quality): def process_image_pyvips(path, *, maxsize, quality):
import pyvips
t_start = perf_counter() t_start = perf_counter()
img = pyvips.Image.new_from_file(str(path), access="sequential") img = pyvips.Image.new_from_file(str(path), access="sequential")
img = img.autorot() img = img.autorot()
@@ -437,7 +447,7 @@ def process_image_pyvips(path, *, maxsize, quality):
ok=True, ok=True,
mime="image/avif", mime="image/avif",
backend="pyvips", backend="pyvips",
total_ms=round((t_end - t_start) * 1000, 1), timings=[round((t_end - t_start) * 1000, 1)],
) )
@@ -476,10 +486,7 @@ def process_image_pillow(path, *, maxsize, quality):
ok=True, ok=True,
mime="image/avif", mime="image/avif",
backend="pillow", backend="pillow",
load_ms=round(load_ms, 1), timings=[round(load_ms, 1), round(proc_ms, 1), round(save_ms, 1)],
process_ms=round(proc_ms, 1),
save_ms=round(save_ms, 1),
total_ms=round((t_end - t_load) * 1000, 1),
) )
@@ -494,18 +501,29 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
t_load_end = perf_counter() t_load_end = perf_counter()
t_save_start = perf_counter() t_save_start = perf_counter()
if FORCE_PIL:
ret = pix.pil_tobytes( ret = pix.pil_tobytes(
format="avif", quality=quality, speed=10, max_threads=1, avif=1 format="avif", quality=quality, speed=10, max_threads=1, avif=1
) )
backend = "pdf"
else:
img = pyvips.Image.new_from_memory(
pix.samples_mv, pix.width, pix.height, pix.n, "uchar"
)
ret = img.write_to_buffer(
".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True
)
backend = "pdf+pyvips"
t_save_end = perf_counter() t_save_end = perf_counter()
return ret, PreviewResponse( return ret, PreviewResponse(
ok=True, ok=True,
mime="image/avif", mime="image/avif",
backend="pdf", backend=backend,
load_ms=round((t_load_end - t_load_start) * 1000, 1), timings=[
save_ms=round((t_save_end - t_save_start) * 1000, 1), round((t_load_end - t_load_start) * 1000, 1),
total_ms=round((t_save_end - t_load_start) * 1000, 1), round((t_save_end - t_save_start) * 1000, 1),
],
) )
@@ -623,9 +641,10 @@ def process_video(path, *, maxsize, quality):
ok=True, ok=True,
mime="image/avif", mime="image/avif",
backend="video", backend="video",
load_ms=round((t_load_end - t_load_start) * 1000, 1), timings=[
save_ms=round((t_save_end - t_save_start) * 1000, 1), round((t_load_end - t_load_start) * 1000, 1),
total_ms=round((t_save_end - t_load_start) * 1000, 1), round((t_save_end - t_save_start) * 1000, 1),
],
) )
del imgdata, istream, ostream, icc, occ, frame del imgdata, istream, ostream, icc, occ, frame
gc.collect() gc.collect()
cista/preview_worker.py
+6 -8
View File
@@ -8,6 +8,7 @@ Framed response format:
(blake3(packet))(uint32 json size)(uint32 payload size)(json)(binary payload) (blake3(packet))(uint32 json size)(uint32 payload size)(json)(binary payload)
where packet = (uint32 json size)(uint32 payload size)(json)(binary payload). where packet = (uint32 json size)(uint32 payload size)(json)(binary payload).
""" """
import logging import logging
import struct import struct
import sys import sys
@@ -28,10 +29,7 @@ class PreviewResponse(msgspec.Struct, omit_defaults=True):
ok: bool ok: bool
mime: str | None = None mime: str | None = None
backend: str | None = None backend: str | None = None
load_ms: float | None = None timings: list[float] | None = None
process_ms: float | None = None
save_ms: float | None = None
total_ms: float | None = None
error: str | None = None error: str | None = None
@@ -50,9 +48,7 @@ def _write_response(resp: PreviewResponse, payload: bytes) -> None:
def _run_once() -> None: def _run_once() -> None:
if len(sys.argv) != 5: if len(sys.argv) != 5:
sys.stderr.write( sys.stderr.write(f"Usage: {sys.argv[0]} <path> <quality> <maxsize> <maxzoom>\n")
f"Usage: {sys.argv[0]} <path> <quality> <maxsize> <maxzoom>\n"
)
sys.exit(1) sys.exit(1)
from cista.preview import dispatch from cista.preview import dispatch
@@ -76,7 +72,9 @@ def _run_loop() -> None:
return return
try: try:
req = _dec_req.decode(line) req = _dec_req.decode(line)
result, resp = dispatch(Path(req.path), req.quality, req.maxsize, req.maxzoom) result, resp = dispatch(
Path(req.path), req.quality, req.maxsize, req.maxzoom
)
_write_response(resp, result or b"") _write_response(resp, result or b"")
except Exception as e: except Exception as e:
_write_response(PreviewResponse(ok=False, error=str(e)), b"") _write_response(PreviewResponse(ok=False, error=str(e)), b"")
scripts/preview_images.py
+18 -18
View File
@@ -36,16 +36,16 @@ def main() -> int:
if not folder.is_dir(): if not folder.is_dir():
raise SystemExit(f"Not a directory: {folder}") raise SystemExit(f"Not a directory: {folder}")
files = sorted(path for path in folder.rglob("*") if path.is_file() and is_image_file(path)) files = sorted(
path for path in folder.rglob("*") if path.is_file() and is_image_file(path)
)
if not files: if not files:
print(f"No image files found under {folder}") print(f"No image files found under {folder}")
return 0 return 0
total_files = 0 total_files = 0
total_bytes = 0 total_bytes = 0
total_load_ms: float = 0.0 total_timing_slots: list[float] = []
total_process_ms: float = 0.0
total_save_ms: float = 0.0
total_preview_ms: float = 0.0 total_preview_ms: float = 0.0
failures = 0 failures = 0
@@ -67,18 +67,19 @@ def main() -> int:
continue continue
total_bytes += len(preview) total_bytes += len(preview)
total_load_ms += timing.load_ms or 0.0 timings = timing.timings or []
total_process_ms += timing.process_ms or 0.0 if len(total_timing_slots) < len(timings):
total_save_ms += timing.save_ms or 0.0 total_timing_slots.extend([0.0] * (len(timings) - len(total_timing_slots)))
total_preview_ms += timing.total_ms or 0.0 for i, value in enumerate(timings):
total_timing_slots[i] += value
total_ms = sum(timings)
total_preview_ms += total_ms
if timing.load_ms is not None: detail = " / ".join(f"{value:.1f}ms" for value in timings)
detail = ( if detail:
f"load={timing.load_ms:.1f}ms process={timing.process_ms:.1f}ms " detail = f"timings={detail} total={total_ms:.1f}ms"
f"save={timing.save_ms:.1f}ms total={timing.total_ms:.1f}ms"
)
else: else:
detail = f"total={timing.total_ms:.1f}ms" detail = f"total={total_ms:.1f}ms"
print(f"OK {rel} backend={timing.backend} bytes={len(preview)} {detail}") print(f"OK {rel} backend={timing.backend} bytes={len(preview)} {detail}")
completed = total_files - failures completed = total_files - failures
@@ -89,10 +90,9 @@ def main() -> int:
print(f" failed={failures}") print(f" failed={failures}")
print(f" preview_bytes={total_bytes}") print(f" preview_bytes={total_bytes}")
if completed: if completed:
if total_load_ms or total_process_ms or total_save_ms: if total_timing_slots:
print(f" load_total_ms={total_load_ms:.1f}") for i, value in enumerate(total_timing_slots, start=1):
print(f" process_total_ms={total_process_ms:.1f}") print(f" timing{i}_total_ms={value:.1f}")
print(f" save_total_ms={total_save_ms:.1f}")
print(f" preview_total_ms={total_preview_ms:.1f}") print(f" preview_total_ms={total_preview_ms:.1f}")
print(f" preview_avg_ms={total_preview_ms / completed:.1f}") print(f" preview_avg_ms={total_preview_ms / completed:.1f}")
return 0 if failures == 0 else 1 return 0 if failures == 0 else 1