Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
953ec628a0 | ||
|
|
e678c8c267 | ||
|
|
69d58f99e3 | ||
|
|
36764885ed | ||
|
|
5a82560cf2 |
+40
-17
@@ -17,6 +17,7 @@ import gc
|
|||||||
import io
|
import io
|
||||||
import logging
|
import logging
|
||||||
import mimetypes
|
import mimetypes
|
||||||
|
import os
|
||||||
import shlex
|
import shlex
|
||||||
import struct
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -33,9 +34,19 @@ import pyvips
|
|||||||
from blake3 import blake3
|
from blake3 import blake3
|
||||||
|
|
||||||
from cista import config
|
from cista import config
|
||||||
|
from cista.util.logformat import format_level_prefix
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class _WorkerLogFormatter(logging.Formatter):
|
||||||
|
"""Emoji level prefix like the main process, tagged with the worker pid."""
|
||||||
|
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
prefix = format_level_prefix(record.levelno)
|
||||||
|
return f"{prefix}worker[{os.getpid()}]: {record.getMessage()}"
|
||||||
|
|
||||||
|
|
||||||
AVIF_FAST_EFFORT = 0
|
AVIF_FAST_EFFORT = 0
|
||||||
|
|
||||||
DOC_PREVIEW_SUFFIXES = {".pdf", ".xps", ".epub", ".mobi"}
|
DOC_PREVIEW_SUFFIXES = {".pdf", ".xps", ".epub", ".mobi"}
|
||||||
@@ -202,6 +213,13 @@ def _image_via_ffmpeg(path: Path, maxsize: int, quality: int) -> bytes:
|
|||||||
tmp_path = tmp_f.name
|
tmp_path = tmp_f.name
|
||||||
cmd = [
|
cmd = [
|
||||||
"ffmpeg",
|
"ffmpeg",
|
||||||
|
# Keep error messages, drop the banner/config/stream-mapping spam.
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostats",
|
||||||
|
# No interactive keyboard prompts ("Press [q] to stop ...").
|
||||||
|
"-nostdin",
|
||||||
"-y",
|
"-y",
|
||||||
"-i",
|
"-i",
|
||||||
str(path),
|
str(path),
|
||||||
@@ -222,8 +240,9 @@ def _image_via_ffmpeg(path: Path, maxsize: int, quality: int) -> bytes:
|
|||||||
new_w = int(w * scale)
|
new_w = int(w * scale)
|
||||||
new_h = int(h * scale)
|
new_h = int(h * scale)
|
||||||
# insert -s <wxh> right after the input file
|
# insert -s <wxh> right after the input file
|
||||||
cmd.insert(4, "-s")
|
input_index = cmd.index(str(path)) + 1
|
||||||
cmd.insert(5, f"{new_w}x{new_h}")
|
cmd.insert(input_index, "-s")
|
||||||
|
cmd.insert(input_index + 1, f"{new_w}x{new_h}")
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
# stdin=DEVNULL is critical: ffmpeg must not inherit the worker's
|
# stdin=DEVNULL is critical: ffmpeg must not inherit the worker's
|
||||||
@@ -243,10 +262,10 @@ def _image_via_ffmpeg(path: Path, maxsize: int, quality: int) -> bytes:
|
|||||||
stderr = (e.stderr or b"").decode(errors="replace").strip()
|
stderr = (e.stderr or b"").decode(errors="replace").strip()
|
||||||
if stderr:
|
if stderr:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"ffmpeg failed (exit {e.returncode}): {shell_cmd}\n{stderr}"
|
f"ffmpeg failed (exit {e.returncode}):\n{shell_cmd}\n{stderr}"
|
||||||
) from e
|
) from e
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"ffmpeg failed (exit {e.returncode}): {shell_cmd}"
|
f"ffmpeg failed (exit {e.returncode}):\n{shell_cmd}"
|
||||||
) from e
|
) from e
|
||||||
with Path(tmp_path).open("rb") as f:
|
with Path(tmp_path).open("rb") as f:
|
||||||
return f.read()
|
return f.read()
|
||||||
@@ -274,9 +293,10 @@ def process_image_pyvips(path, *, maxsize, quality):
|
|||||||
height=height,
|
height=height,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Other image formats: pyvips first, ffmpeg fallback.
|
# Other image formats: pyvips only. ffmpeg is not a useful fallback
|
||||||
|
# here — when pyvips cannot decode a file, ffmpeg's image decoders
|
||||||
|
# cannot either, and their failure output is far noisier.
|
||||||
load_opts = {"access": "sequential"}
|
load_opts = {"access": "sequential"}
|
||||||
orig_w = orig_h = None
|
|
||||||
try:
|
try:
|
||||||
img = pyvips.Image.new_from_file(str(path), **load_opts)
|
img = pyvips.Image.new_from_file(str(path), **load_opts)
|
||||||
img = img.autorot()
|
img = img.autorot()
|
||||||
@@ -288,13 +308,11 @@ def process_image_pyvips(path, *, maxsize, quality):
|
|||||||
".avif",
|
".avif",
|
||||||
Q=quality,
|
Q=quality,
|
||||||
effort=AVIF_FAST_EFFORT,
|
effort=AVIF_FAST_EFFORT,
|
||||||
strip=True,
|
keep="none",
|
||||||
)
|
)
|
||||||
backend = "pyvips"
|
except pyvips.error.Error as e:
|
||||||
except pyvips.error.Error:
|
raise ValueError(f"cannot decode image: {e}") from e
|
||||||
orig_w, orig_h = None, None
|
backend = "pyvips"
|
||||||
ret = _image_via_ffmpeg(path, maxsize, quality)
|
|
||||||
backend = "ffmpeg"
|
|
||||||
t_end = perf_counter()
|
t_end = perf_counter()
|
||||||
|
|
||||||
return ret, PreviewResponse(
|
return ret, PreviewResponse(
|
||||||
@@ -320,7 +338,7 @@ def process_image_buffer(data: bytes, *, quality, maxsize, maxzoom):
|
|||||||
".avif",
|
".avif",
|
||||||
Q=quality,
|
Q=quality,
|
||||||
effort=AVIF_FAST_EFFORT,
|
effort=AVIF_FAST_EFFORT,
|
||||||
strip=True,
|
keep="none",
|
||||||
)
|
)
|
||||||
t_end = perf_counter()
|
t_end = perf_counter()
|
||||||
|
|
||||||
@@ -348,7 +366,7 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
|||||||
img = pyvips.Image.new_from_memory(
|
img = pyvips.Image.new_from_memory(
|
||||||
pix.samples_mv, pix.width, pix.height, pix.n, "uchar"
|
pix.samples_mv, pix.width, pix.height, pix.n, "uchar"
|
||||||
)
|
)
|
||||||
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True)
|
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, keep="none")
|
||||||
backend = "pdf+pyvips"
|
backend = "pdf+pyvips"
|
||||||
t_save_end = perf_counter()
|
t_save_end = perf_counter()
|
||||||
|
|
||||||
@@ -554,11 +572,16 @@ def _run_loop() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
# Configure all log output to stderr before any imports that may emit logs.
|
# Configure all log output to stderr before any imports that may emit
|
||||||
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
|
# logs. stderr is inherited by the parent, so this lands in the server
|
||||||
|
# log, formatted like the main process and tagged with the worker pid.
|
||||||
|
handler = logging.StreamHandler(sys.stderr)
|
||||||
|
handler.setFormatter(_WorkerLogFormatter())
|
||||||
|
logging.basicConfig(level=logging.INFO, handlers=[handler])
|
||||||
|
# pyvips is chatty at INFO ("threadpool completed ..." per operation).
|
||||||
|
logging.getLogger("pyvips").setLevel(logging.WARNING)
|
||||||
try:
|
try:
|
||||||
config.load_config()
|
config.load_config()
|
||||||
logger.info("preview-worker config=%s", config.conffile)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("preview-worker failed to load config at startup")
|
logger.exception("preview-worker failed to load config at startup")
|
||||||
if len(sys.argv) > 1:
|
if len(sys.argv) > 1:
|
||||||
|
|||||||
+3
-31
@@ -3,11 +3,13 @@
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unicodedata
|
|
||||||
from ipaddress import IPv6Address
|
from ipaddress import IPv6Address
|
||||||
|
|
||||||
from sanic.log import LOGGING_CONFIG_DEFAULTS
|
from sanic.log import LOGGING_CONFIG_DEFAULTS
|
||||||
|
|
||||||
|
from cista.util.logformat import EmojiFormatter as _EmojiFormatter
|
||||||
|
from cista.util.logformat import display_width as _display_width
|
||||||
|
|
||||||
logger = logging.getLogger("cista.access")
|
logger = logging.getLogger("cista.access")
|
||||||
|
|
||||||
|
|
||||||
@@ -132,14 +134,6 @@ def format_duration_ms(duration_ms: float) -> str:
|
|||||||
return f"{hours}h{minutes}m"
|
return f"{hours}h{minutes}m"
|
||||||
|
|
||||||
|
|
||||||
def _display_width(text: str) -> int:
|
|
||||||
return sum(
|
|
||||||
1 + (unicodedata.east_asian_width(c) in "FW")
|
|
||||||
for c in text
|
|
||||||
if unicodedata.category(c) != "Mn"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _format_left(label: str) -> str:
|
def _format_left(label: str) -> str:
|
||||||
return label[:19].ljust(19)
|
return label[:19].ljust(19)
|
||||||
|
|
||||||
@@ -279,28 +273,6 @@ def configure_access_logging() -> None:
|
|||||||
logger.propagate = False
|
logger.propagate = False
|
||||||
|
|
||||||
|
|
||||||
_LEVEL_EMOJI = {
|
|
||||||
logging.DEBUG: "🔍",
|
|
||||||
logging.INFO: "ℹ️", # noqa: RUF001
|
|
||||||
logging.WARNING: "⚠️",
|
|
||||||
logging.ERROR: "🛑",
|
|
||||||
logging.CRITICAL: "🛑",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _format_level_prefix(levelno: int) -> str:
|
|
||||||
emoji = _LEVEL_EMOJI.get(levelno, "▪️")
|
|
||||||
prefix = f"{emoji} "
|
|
||||||
return prefix + (" " * max(0, 3 - _display_width(prefix)))
|
|
||||||
|
|
||||||
|
|
||||||
class _EmojiFormatter(logging.Formatter):
|
|
||||||
"""Compact formatter: emoji + message, no timestamp/level text/logger name."""
|
|
||||||
|
|
||||||
def format(self, record: logging.LogRecord) -> str:
|
|
||||||
return _format_level_prefix(record.levelno) + record.getMessage()
|
|
||||||
|
|
||||||
|
|
||||||
def configure_main_logging() -> None:
|
def configure_main_logging() -> None:
|
||||||
"""Replace Sanic's verbose 'Main yyyy-mm-dd INFO:' prefix with emoji-only format.
|
"""Replace Sanic's verbose 'Main yyyy-mm-dd INFO:' prefix with emoji-only format.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""Shared log formatting helpers with no Sanic dependency.
|
||||||
|
|
||||||
|
Used by the main process (cista.sanic_logging) and by the preview worker
|
||||||
|
subprocess, which must not import Sanic.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
LEVEL_EMOJI = {
|
||||||
|
logging.DEBUG: "🔍",
|
||||||
|
logging.INFO: "ℹ️", # noqa: RUF001
|
||||||
|
logging.WARNING: "⚠️",
|
||||||
|
logging.ERROR: "🛑",
|
||||||
|
logging.CRITICAL: "🛑",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def display_width(text: str) -> int:
|
||||||
|
return sum(
|
||||||
|
1 + (unicodedata.east_asian_width(c) in "FW")
|
||||||
|
for c in text
|
||||||
|
if unicodedata.category(c) != "Mn"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def format_level_prefix(levelno: int) -> str:
|
||||||
|
emoji = LEVEL_EMOJI.get(levelno, "▪️")
|
||||||
|
prefix = f"{emoji} "
|
||||||
|
return prefix + (" " * max(0, 3 - display_width(prefix)))
|
||||||
|
|
||||||
|
|
||||||
|
class EmojiFormatter(logging.Formatter):
|
||||||
|
"""Compact formatter: emoji + message, no timestamp/level text/logger name."""
|
||||||
|
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
return format_level_prefix(record.levelno) + record.getMessage()
|
||||||
Reference in New Issue
Block a user