Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4103b8928c | ||
|
|
3c1894f337 | ||
|
|
7ac373179b | ||
|
|
a99996bd9c |
@@ -2,16 +2,16 @@
|
||||
|
||||
Usage:
|
||||
mediapreview <path> [-o OUTPUT] [-q QUALITY] [--maxsize N] [--maxzoom Z]
|
||||
mediapreview oosetup [<name>] [<port>]
|
||||
mediapreview oosetup [<name>]
|
||||
mediapreview (-h | --help)
|
||||
|
||||
Generate an AVIF preview for a media file (one-shot, in-process), or set up
|
||||
the bundled OnlyOffice container.
|
||||
the bundled OnlyOffice container (isolated network, reachable from the host
|
||||
at its fixed container IP).
|
||||
|
||||
Arguments:
|
||||
<path> media file to preview
|
||||
<name> container name [default: onlyoffice-mediapreview]
|
||||
<port> container host port [default: 8988]
|
||||
|
||||
Options:
|
||||
-o OUTPUT output .avif file (default: write AVIF bytes to stdout)
|
||||
@@ -43,7 +43,7 @@ def _configure_logging() -> None:
|
||||
logging.getLogger("pyvips").setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def _oosetup(name: str, port: int) -> None:
|
||||
def _oosetup(name: str) -> None:
|
||||
try:
|
||||
# Lazy import: keeps the base CLI free of office-extra concerns.
|
||||
from mediapreview.office import setup_docker # noqa: PLC0415
|
||||
@@ -52,7 +52,7 @@ def _oosetup(name: str, port: int) -> None:
|
||||
sys.exit(1)
|
||||
try:
|
||||
# Logs go to stderr; stdout carries only the secret line below.
|
||||
secret = setup_docker(name=name, port=port)
|
||||
secret = setup_docker(name=name)
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"error: OnlyOffice setup failed: {e}\n")
|
||||
sys.exit(1)
|
||||
@@ -102,13 +102,8 @@ def main() -> None:
|
||||
# by the <path> pattern if it came second. Dispatch it before parsing;
|
||||
# the main help above still documents both modes.
|
||||
if sys.argv[1:2] == ["oosetup"]:
|
||||
args = docopt(
|
||||
"Usage:\n mediapreview oosetup [<name>] [<port>]", argv=sys.argv[1:]
|
||||
)
|
||||
_oosetup(
|
||||
args["<name>"] or "onlyoffice-mediapreview",
|
||||
int(args["<port>"] or 8988),
|
||||
)
|
||||
args = docopt("Usage:\n mediapreview oosetup [<name>]", argv=sys.argv[1:])
|
||||
_oosetup(args["<name>"] or "onlyoffice-mediapreview")
|
||||
return
|
||||
_preview(docopt(__doc__))
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ def dispatch(path, quality, maxsize, maxzoom, data=None):
|
||||
backend = "unknown"
|
||||
try:
|
||||
if data:
|
||||
backend = "pyvips"
|
||||
backend = "vips"
|
||||
return process_image_buffer(
|
||||
data, quality=quality, maxsize=maxsize, maxzoom=maxzoom
|
||||
)
|
||||
@@ -47,11 +47,11 @@ def dispatch(path, quality, maxsize, maxzoom, data=None):
|
||||
backend = "video"
|
||||
return process_video(path, quality=quality, maxsize=maxsize)
|
||||
if mime_type and mime_type.startswith("image/"):
|
||||
backend = "pyvips"
|
||||
backend = "vips"
|
||||
return process_image(path, quality=quality, maxsize=maxsize)
|
||||
except PreviewError:
|
||||
# Already structured (e.g. a stage of a combined pipeline like
|
||||
# pdf+pyvips) — keep the original backend/stage identity.
|
||||
# Already structured (e.g. a failing stage of a combined pipeline
|
||||
# like pdf+vips) — keep the original backend identity.
|
||||
raise
|
||||
except ValueError as e:
|
||||
raise backend_error(backend, str(e)) from e
|
||||
|
||||
@@ -148,7 +148,7 @@ def process_image_pyvips(path, *, maxsize, quality):
|
||||
)
|
||||
except pyvips.error.Error as e:
|
||||
raise ValueError(f"cannot decode image: {e}") from e
|
||||
backend = "pyvips"
|
||||
backend = "vips"
|
||||
t_end = perf_counter()
|
||||
|
||||
return ret, PreviewResponse(
|
||||
@@ -181,7 +181,7 @@ def process_image_buffer(data: bytes, *, quality, maxsize, maxzoom):
|
||||
return ret, PreviewResponse(
|
||||
ok=True,
|
||||
mime="image/avif",
|
||||
backend="pyvips",
|
||||
backend="vips",
|
||||
timings=[round((t_end - t_start) * 1000, 1)],
|
||||
width=orig_w,
|
||||
height=orig_h,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""PDF/XPS/EPUB preview conversion via PyMuPDF + pyvips."""
|
||||
"""PDF/XPS/EPUB preview conversion via PyMuPDF + vips."""
|
||||
|
||||
from time import perf_counter
|
||||
|
||||
@@ -13,7 +13,7 @@ try:
|
||||
except ImportError: # pragma: no cover - optional pdf extra
|
||||
pymupdf = None
|
||||
|
||||
BACKEND = "pdf+pyvips"
|
||||
BACKEND = "pdf+vips"
|
||||
|
||||
|
||||
def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
||||
@@ -31,7 +31,8 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
||||
pix = page.get_pixmap(matrix=mat)
|
||||
samples, width, height, n = pix.samples_mv, pix.width, pix.height, pix.n
|
||||
except Exception as e:
|
||||
raise backend_error(BACKEND, str(e), stage="pdf") from e
|
||||
# vips was never reached — this is a plain pdf error.
|
||||
raise backend_error("pdf", str(e)) from e
|
||||
t_load_end = perf_counter()
|
||||
|
||||
t_save_start = perf_counter()
|
||||
@@ -39,7 +40,7 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
||||
img = pyvips.Image.new_from_memory(samples, width, height, n, "uchar")
|
||||
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, keep="none")
|
||||
except Exception as e:
|
||||
raise backend_error(BACKEND, str(e), stage="pyvips") from e
|
||||
raise backend_error(BACKEND, str(e)) from e
|
||||
t_save_end = perf_counter()
|
||||
|
||||
return ret, PreviewResponse(
|
||||
|
||||
@@ -36,10 +36,17 @@ RUN apt-get update -qq && \
|
||||
ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Clone the open-source server components (shallow, ~15 MB).
|
||||
# The master branch is used because the Linux/web tags are not published
|
||||
# in the server repo; the license.js file has been stable for years.
|
||||
RUN git clone --depth 1 https://github.com/ONLYOFFICE/server.git /opt/oo-server
|
||||
# Pin the open-source server components to a known-good commit (~15 MB).
|
||||
# The master branch is a moving target (the Linux/web tags are not published
|
||||
# in the server repo): a 2026 convertermaster change there detects community
|
||||
# edition as a "memory runtime" and forks NO converter workers, silently
|
||||
# breaking all conversions. Pinned to the commit proven in production.
|
||||
ARG ONLYOFFICE_SERVER_REF=4e56b8d056640557ffcd8c860a65535ab6cbd95b
|
||||
RUN git init /opt/oo-server && \
|
||||
cd /opt/oo-server && \
|
||||
git remote add origin https://github.com/ONLYOFFICE/server.git && \
|
||||
git fetch --depth 1 origin "$ONLYOFFICE_SERVER_REF" && \
|
||||
git checkout FETCH_HEAD
|
||||
|
||||
# Patch license.js so the converter worker count is read from an env var
|
||||
# instead of being hardcoded to 1.
|
||||
@@ -47,6 +54,12 @@ RUN sed -i \
|
||||
's/count: 1,/count: parseInt(process.env.WORKERS, 10) || 8,/' \
|
||||
/opt/oo-server/Common/sources/license.js
|
||||
|
||||
# Defense in depth: never take the "memory runtime" branch that forks no
|
||||
# converter workers, even if the pinned ref is bumped carelessly.
|
||||
RUN sed -i \
|
||||
's/runtimeProfile\.isMemoryRuntime()/false \/* patched: always fork converter workers *\//g' \
|
||||
/opt/oo-server/FileConverter/sources/convertermaster.js
|
||||
|
||||
# Install npm dependencies for the modules the FileConverter touches.
|
||||
# DocService deps are also needed because converter.js pulls in baseConnector.
|
||||
RUN cd /opt/oo-server/Common && npm ci --no-audit --no-fund
|
||||
|
||||
+27
-19
@@ -17,9 +17,10 @@ The hierarchy is intentionally small:
|
||||
|
||||
- ``OnlyOfficeError`` covers all OnlyOffice failures; optional fields
|
||||
(``code``, ``status``, ``url``, ``snippet``) describe the specific failure.
|
||||
- ``PreviewBackendError`` covers backend conversion failures (ffmpeg, pyvips,
|
||||
pdf, etc.); ``stage`` identifies the failing step of a combined pipeline
|
||||
(e.g. "pdf" vs "pyvips" in the "pdf+pyvips" backend).
|
||||
- ``PreviewBackendError`` covers backend conversion failures (ffmpeg, vips,
|
||||
pdf, etc.). Combined pipelines report the failing step in ``backend``
|
||||
(e.g. "pdf" if pdf reading failed before vips was reached, "pdf+vips"
|
||||
for a vips write failure).
|
||||
- ``PreviewTimeoutError`` covers timeouts for any backend.
|
||||
- ``PreviewCancelledError`` covers cancellations (e.g. pool shutdown).
|
||||
|
||||
@@ -80,17 +81,6 @@ class OnlyOfficeError(PreviewError):
|
||||
class PreviewBackendError(PreviewError):
|
||||
"""Backend conversion failure (image/video/pdf/etc)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str = "preview failed",
|
||||
short: str = "error",
|
||||
*,
|
||||
stage: str | None = None,
|
||||
backend: str | None = None,
|
||||
):
|
||||
super().__init__(message, short, backend=backend)
|
||||
self.stage = stage
|
||||
|
||||
|
||||
class PreviewTimeoutError(PreviewError):
|
||||
"""Preview conversion exceeded its timeout for a given backend."""
|
||||
@@ -102,9 +92,11 @@ class PreviewTimeoutError(PreviewError):
|
||||
*,
|
||||
timeout_seconds: float = 0.0,
|
||||
backend: str | None = None,
|
||||
fetched: bool | None = None,
|
||||
):
|
||||
super().__init__(message, short, backend=backend)
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.fetched = fetched
|
||||
|
||||
|
||||
class PreviewCancelledError(PreviewError):
|
||||
@@ -163,22 +155,38 @@ def onlyoffice_no_fileurl_error(snippet: str | None = None) -> OnlyOfficeError:
|
||||
return OnlyOfficeError(log, "no-fileurl error", snippet=snippet)
|
||||
|
||||
|
||||
def backend_error(backend: str, message: str, *, stage: str | None = None) -> PreviewBackendError:
|
||||
short = message.splitlines()[0][:60]
|
||||
def backend_error(backend: str, message: str) -> PreviewBackendError:
|
||||
short = message.splitlines()[0]
|
||||
# Many backend messages look like "source: summary: detail ...".
|
||||
# Drop the source prefix and any trailing detail so the short label
|
||||
# is usable in UIs with limited space.
|
||||
if ": " in short:
|
||||
short = short.split(": ", 1)[1]
|
||||
if ": " in short:
|
||||
short = short.split(": ", 1)[0]
|
||||
short = short[:60]
|
||||
return PreviewBackendError(
|
||||
f"[{backend}] preview failed: {message}",
|
||||
short,
|
||||
backend=backend,
|
||||
stage=stage,
|
||||
)
|
||||
|
||||
|
||||
def preview_timeout_error(backend: str, timeout_seconds: float) -> PreviewTimeoutError:
|
||||
def preview_timeout_error(
|
||||
backend: str, timeout_seconds: float, fetched: bool | None = None
|
||||
) -> PreviewTimeoutError:
|
||||
log = f"{backend.capitalize()} preview timed out after {timeout_seconds}s"
|
||||
if fetched is not None:
|
||||
# OnlyOffice: whether it ever downloaded the input file from our
|
||||
# callback server distinguishes network/callback failures from a
|
||||
# stalled conversion.
|
||||
log += " (input file fetched)" if fetched else " (input file never fetched)"
|
||||
return PreviewTimeoutError(
|
||||
f"{backend.capitalize()} preview timed out after {timeout_seconds}s",
|
||||
log,
|
||||
"timeout",
|
||||
backend=backend,
|
||||
timeout_seconds=timeout_seconds,
|
||||
fetched=fetched,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -90,5 +90,5 @@ def expected_backend(path: Path) -> str:
|
||||
if mime_type and mime_type.startswith("video/"):
|
||||
return "video"
|
||||
if mime_type and mime_type.startswith("image/"):
|
||||
return "pyvips"
|
||||
return "vips"
|
||||
return "preview"
|
||||
|
||||
+137
-27
@@ -47,6 +47,14 @@ except ImportError: # pragma: no cover - optional office extra
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Isolated docker network for the OnlyOffice container: internal-only (no
|
||||
# outbound internet), the container can only reach the host on this bridge.
|
||||
# Docker discards published ports on internal networks, so the container is
|
||||
# reached at its fixed IP instead of a published localhost port.
|
||||
OO_NETWORK = "oonet"
|
||||
OO_SUBNET = "172.30.0.0/24"
|
||||
OO_CONTAINER_IP = "172.30.0.2"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -57,22 +65,57 @@ _httpx_client_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
|
||||
def _get_onlyoffice_url() -> str:
|
||||
return os.environ.get(
|
||||
"ONLYOFFICE_URL",
|
||||
os.environ.get("ONLYOFFICE_CISTA_URL", "http://localhost:8988"),
|
||||
)
|
||||
if url := os.environ.get(
|
||||
"ONLYOFFICE_URL", os.environ.get("ONLYOFFICE_CISTA_URL")
|
||||
):
|
||||
return url
|
||||
# When the isolated network exists, the container is at its fixed IP and
|
||||
# no localhost port is published (Docker discards ports on internal
|
||||
# networks). Otherwise assume a legacy setup with a published port.
|
||||
if _docker_network_gateway(OO_NETWORK):
|
||||
return f"http://{OO_CONTAINER_IP}"
|
||||
return "http://localhost:8988"
|
||||
|
||||
|
||||
def _get_jwt_secret() -> str:
|
||||
return os.environ.get("ONLYOFFICE_JWT_SECRET", "")
|
||||
|
||||
|
||||
def _docker_network_gateway(network: str) -> str | None:
|
||||
"""Return the host-side gateway IP of a docker network, or None."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"network",
|
||||
"inspect",
|
||||
network,
|
||||
"--format",
|
||||
"{{range .IPAM.Config}}{{.Gateway}}{{end}}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
check=False,
|
||||
)
|
||||
gateway = result.stdout.strip()
|
||||
if result.returncode == 0 and gateway:
|
||||
return gateway
|
||||
except Exception:
|
||||
logger.debug("Failed to inspect docker network %s", network)
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_callback_host() -> str:
|
||||
"""Return the host IP that OnlyOffice (usually in Docker) can use to reach us."""
|
||||
"""Return the host IP that OnlyOffice (in Docker) can use to reach us."""
|
||||
if host := os.environ.get("ONLYOFFICE_CALLBACK_HOST"):
|
||||
return host
|
||||
# Try to auto-detect docker bridge IP
|
||||
# Prefer the gateway of the isolated network setup_docker() creates —
|
||||
# this is the network the container is actually attached to.
|
||||
if gateway := _docker_network_gateway(OO_NETWORK):
|
||||
return gateway
|
||||
# Fall back to the default docker bridge IP
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["/sbin/ip", "-4", "addr", "show", "docker0"],
|
||||
@@ -156,12 +199,14 @@ def log_reachable_info() -> None:
|
||||
logger.warning("OnlyOffice probe failed%s", suffix)
|
||||
|
||||
|
||||
def setup_docker(name: str = "onlyoffice-mediapreview", port: int = 8988) -> str:
|
||||
def setup_docker(name: str = "onlyoffice-mediapreview") -> str:
|
||||
"""Build and run the patched OnlyOffice Docker image.
|
||||
|
||||
Uses ONLYOFFICE_JWT_SECRET if set, otherwise generates a random secret.
|
||||
Returns the secret used, so the caller is responsible for persisting it
|
||||
(the CLI prints it as `ONLYOFFICE_JWT_SECRET=<token>`).
|
||||
The container runs on an isolated internal network (OO_NETWORK) with no
|
||||
outbound internet and no published ports; the host reaches it at
|
||||
OO_CONTAINER_IP. Uses ONLYOFFICE_JWT_SECRET if set, otherwise generates a
|
||||
random secret. Returns the secret used, so the caller is responsible for
|
||||
persisting it (the CLI prints it as `ONLYOFFICE_JWT_SECRET=<token>`).
|
||||
The Docker build context ships inside the package at `mediapreview/docker`.
|
||||
"""
|
||||
if secret := _get_jwt_secret():
|
||||
@@ -182,13 +227,33 @@ def setup_docker(name: str = "onlyoffice-mediapreview", port: int = 8988) -> str
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("Failed to build OnlyOffice image")
|
||||
|
||||
# Isolated network: internal-only, so the container has no outbound
|
||||
# internet access and can only reach the host on this bridge (needed
|
||||
# for the preview file callback). Already-exists is fine.
|
||||
net_cmd = [
|
||||
"docker",
|
||||
"network",
|
||||
"create",
|
||||
"--internal",
|
||||
"--subnet",
|
||||
OO_SUBNET,
|
||||
OO_NETWORK,
|
||||
]
|
||||
result = subprocess.run(net_cmd, capture_output=True, check=False) # noqa: S603
|
||||
if result.returncode != 0 and b"already exists" not in result.stderr:
|
||||
raise RuntimeError(
|
||||
f"Failed to create docker network {OO_NETWORK}: {result.stderr.decode(errors='replace').strip()}"
|
||||
)
|
||||
|
||||
logger.info("Starting OnlyOffice container")
|
||||
run_cmd = [
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"-p",
|
||||
f"{port}:80",
|
||||
"--network",
|
||||
OO_NETWORK,
|
||||
"--ip",
|
||||
OO_CONTAINER_IP,
|
||||
"-e",
|
||||
f"JWT_SECRET={secret}",
|
||||
"-e",
|
||||
@@ -203,7 +268,12 @@ def setup_docker(name: str = "onlyoffice-mediapreview", port: int = 8988) -> str
|
||||
result = subprocess.run(run_cmd, check=False, shell=False) # noqa: S603
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("Failed to start OnlyOffice container")
|
||||
logger.info("OnlyOffice is running on http://localhost:%d", port)
|
||||
# Docker discards published ports on internal networks, so the container
|
||||
# is reached at its fixed IP; no localhost port is exposed.
|
||||
logger.info("OnlyOffice is running on http://%s", OO_CONTAINER_IP)
|
||||
logger.info(
|
||||
"Callback host for file downloads: %s", _docker_network_gateway(OO_NETWORK)
|
||||
)
|
||||
return secret
|
||||
|
||||
|
||||
@@ -260,7 +330,9 @@ async def is_available_cached() -> bool:
|
||||
|
||||
class _QuietHandler(SimpleHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args) -> None:
|
||||
pass
|
||||
# Any request logged here means a client (OnlyOffice) connected to
|
||||
# fetch the file; record it for timeout diagnostics.
|
||||
self.server.oo_fetched = True
|
||||
|
||||
|
||||
def _get_free_port() -> int:
|
||||
@@ -277,6 +349,7 @@ def _serve_file_temporarily(file_path: Path):
|
||||
|
||||
handler = partial(_QuietHandler, directory=directory)
|
||||
httpd = socketserver.TCPServer(("0.0.0.0", port), handler) # noqa: S104
|
||||
httpd.oo_fetched = False
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
@@ -297,9 +370,14 @@ def _build_jwt_token(payload: dict) -> str | None:
|
||||
return jwt.encode(payload, secret, algorithm="HS256")
|
||||
|
||||
|
||||
async def convert_to_png_async(file_path: Path, request_timeout: float = 5.0) -> bytes:
|
||||
async def convert_to_png_async(
|
||||
file_path: Path, request_timeout: float = 7.0, download_timeout: float = 2.0
|
||||
) -> bytes:
|
||||
"""Convert *file_path* to PNG using OnlyOffice Document Server (async).
|
||||
|
||||
With ``async: false`` the conversion itself runs inside the POST request,
|
||||
so *request_timeout* must cover full conversion time. *download_timeout*
|
||||
covers fetching the resulting one-page PNG, which is pure transfer.
|
||||
Returns the PNG bytes. Raises RuntimeError on failure.
|
||||
"""
|
||||
if httpx is None or jwt is None:
|
||||
@@ -340,7 +418,9 @@ async def convert_to_png_async(file_path: Path, request_timeout: float = 5.0) ->
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.TimeoutException as e:
|
||||
raise preview_timeout_error("onlyoffice", request_timeout) from e
|
||||
raise preview_timeout_error(
|
||||
"onlyoffice", request_timeout, fetched=httpd.oo_fetched
|
||||
) from e
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise onlyoffice_http_error(e.response.status_code) from e
|
||||
except httpx.RequestError as e:
|
||||
@@ -367,10 +447,10 @@ async def convert_to_png_async(file_path: Path, request_timeout: float = 5.0) ->
|
||||
|
||||
# Download converted PNG
|
||||
try:
|
||||
png_response = await client.get(file_url, timeout=request_timeout)
|
||||
png_response = await client.get(file_url, timeout=download_timeout)
|
||||
png_response.raise_for_status()
|
||||
except httpx.TimeoutException as e:
|
||||
raise preview_timeout_error("onlyoffice", request_timeout) from e
|
||||
raise preview_timeout_error("onlyoffice", download_timeout) from e
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise onlyoffice_http_error(e.response.status_code) from e
|
||||
except httpx.RequestError as e:
|
||||
@@ -391,12 +471,23 @@ async def convert_to_png_async(file_path: Path, request_timeout: float = 5.0) ->
|
||||
OO_MAX_CONCURRENT = max(2, min(8, cpu_count()))
|
||||
|
||||
|
||||
class _InFlight:
|
||||
"""A deduplicated conversion: shared future, its task, and waiter count."""
|
||||
|
||||
__slots__ = ("future", "task", "waiters")
|
||||
|
||||
def __init__(self, future: asyncio.Future[bytes], task: asyncio.Task[None]):
|
||||
self.future = future
|
||||
self.task = task
|
||||
self.waiters = 0
|
||||
|
||||
|
||||
class OOConversionManager:
|
||||
"""Manages async OnlyOffice conversions with deduplication and concurrency limits."""
|
||||
|
||||
def __init__(self, max_concurrent: int = OO_MAX_CONCURRENT):
|
||||
self._semaphore = asyncio.Semaphore(max_concurrent)
|
||||
self._in_flight: dict[str, asyncio.Future[bytes]] = {}
|
||||
self._in_flight: dict[str, _InFlight] = {}
|
||||
self._tasks: set[asyncio.Task[None]] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@@ -408,31 +499,50 @@ class OOConversionManager:
|
||||
key = f"{filepath}:{stat.st_mtime_ns}"
|
||||
|
||||
async with self._lock:
|
||||
if key in self._in_flight:
|
||||
future = self._in_flight[key]
|
||||
else:
|
||||
entry = self._in_flight.get(key)
|
||||
if entry is None:
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self._in_flight[key] = future
|
||||
task = asyncio.create_task(self._do_convert(filepath, key, future))
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
entry = _InFlight(future, task)
|
||||
self._in_flight[key] = entry
|
||||
entry.waiters += 1
|
||||
|
||||
return await future
|
||||
try:
|
||||
# shield: one waiter's cancellation must not cancel the future
|
||||
# shared with other waiters.
|
||||
return await asyncio.shield(entry.future)
|
||||
except asyncio.CancelledError:
|
||||
# The caller hit the (strict) preview deadline or disconnected.
|
||||
# When no other waiter remains, cancel the background task so it
|
||||
# releases its semaphore slot and aborts the HTTP request instead
|
||||
# of running orphaned and piling load onto OnlyOffice.
|
||||
async with self._lock:
|
||||
entry.waiters -= 1
|
||||
orphan = entry.waiters == 0
|
||||
if orphan:
|
||||
entry.task.cancel()
|
||||
raise
|
||||
|
||||
async def _do_convert(
|
||||
self, filepath: Path, key: str, future: asyncio.Future[bytes]
|
||||
) -> None:
|
||||
try:
|
||||
async with self._semaphore:
|
||||
png_bytes = await convert_to_png_async(filepath, request_timeout=5.0)
|
||||
png_bytes = await convert_to_png_async(filepath)
|
||||
except asyncio.CancelledError:
|
||||
# All waiters gave up; cancel the future so nothing hangs on it.
|
||||
if not future.done():
|
||||
future.cancel()
|
||||
raise
|
||||
except Exception as e:
|
||||
if not future.done():
|
||||
future.set_exception(e)
|
||||
async with self._lock:
|
||||
self._in_flight.pop(key, None)
|
||||
else:
|
||||
if not future.done():
|
||||
future.set_result(png_bytes)
|
||||
finally:
|
||||
async with self._lock:
|
||||
self._in_flight.pop(key, None)
|
||||
|
||||
|
||||
@@ -477,7 +477,7 @@ async def generate_office_preview(
|
||||
img, resp = await run_preview(filepath, quality, maxsize, maxzoom, data=png_bytes)
|
||||
|
||||
if resp is not None:
|
||||
resp.backend = "onlyoffice+" + (resp.backend or "pyvips")
|
||||
resp.backend = "onlyoffice+" + (resp.backend or "vips")
|
||||
if resp.timings:
|
||||
resp.timings = [round((t_oo_end - t_oo_start) * 1000, 1), *resp.timings]
|
||||
return img, resp
|
||||
|
||||
@@ -38,6 +38,7 @@ except ImportError: # pragma: no cover - optional worker extra
|
||||
sys.exit(1)
|
||||
|
||||
from mediapreview.backends import dispatch
|
||||
from mediapreview.exceptions import PreviewError
|
||||
from mediapreview.protocol import PreviewRequest, PreviewResponse
|
||||
from mediapreview.util.logformat import format_level_prefix
|
||||
|
||||
@@ -140,7 +141,13 @@ def _run_loop() -> None:
|
||||
)
|
||||
_write_response(resp, result or b"")
|
||||
except Exception as e:
|
||||
logger.exception("Preview worker error for %s", req.path)
|
||||
# PreviewError is an expected failure (broken input, missing
|
||||
# extra, backend error) — a warning suffices. Tracebacks are
|
||||
# reserved for internal errors we did not anticipate.
|
||||
if isinstance(e, PreviewError):
|
||||
logger.warning("Preview failed for %s: %s", req.path, e)
|
||||
else:
|
||||
logger.exception("Preview worker error for %s", req.path)
|
||||
captured = stderr_capture.getvalue().strip()
|
||||
_write_response(
|
||||
PreviewResponse(
|
||||
|
||||
+13
-5
@@ -64,15 +64,23 @@ def test_onlyoffice_no_fileurl_error():
|
||||
assert err.short == "no-fileurl error"
|
||||
|
||||
|
||||
def test_backend_error_stage():
|
||||
"""Combined pipelines tag the failing stage."""
|
||||
err = backend_error("pdf+pyvips", "cannot read document", stage="pdf")
|
||||
assert err.backend == "pdf+pyvips"
|
||||
assert err.stage == "pdf"
|
||||
def test_backend_error_pipeline_backend():
|
||||
"""Combined pipelines report the failing step in the backend name."""
|
||||
err = backend_error("pdf", "cannot read document")
|
||||
assert err.backend == "pdf"
|
||||
assert err.short == "cannot read document"
|
||||
assert isinstance(err, PreviewBackendError)
|
||||
|
||||
|
||||
def test_backend_error_short_message_strips_source_and_detail():
|
||||
"""Backend messages like "source: summary: detail" become just the summary."""
|
||||
err = backend_error(
|
||||
"vips",
|
||||
"pyvips: cannot decode image: unable to load from file b'/mnt/c/Users...",
|
||||
)
|
||||
assert err.short == "cannot decode image"
|
||||
|
||||
|
||||
def test_error_pickle_round_trip():
|
||||
"""Exceptions survive pickling (the worker pool wire) intact."""
|
||||
err = onlyoffice_error_from_code("-8")
|
||||
|
||||
@@ -56,7 +56,7 @@ def _assert_ok(data, resp, backend: str | None = None) -> None:
|
||||
def test_process_image_exif_orientations(path: Path) -> None:
|
||||
"""Every EXIF orientation fixture must produce a valid preview."""
|
||||
data, resp = process_image(path, maxsize=512, quality=60)
|
||||
_assert_ok(data, resp, backend="pyvips")
|
||||
_assert_ok(data, resp, backend="vips")
|
||||
assert resp.width in (1200, 1800)
|
||||
assert resp.height in (1200, 1800)
|
||||
|
||||
@@ -74,7 +74,7 @@ def test_process_image_pyvips() -> None:
|
||||
"""The pyvips-only image backend works on a plain JPEG."""
|
||||
path = FILES / "Landscape_1.jpg"
|
||||
data, resp = process_image_pyvips(path, maxsize=512, quality=60)
|
||||
_assert_ok(data, resp, backend="pyvips")
|
||||
_assert_ok(data, resp, backend="vips")
|
||||
|
||||
|
||||
def test_process_image_buffer() -> None:
|
||||
@@ -83,7 +83,7 @@ def test_process_image_buffer() -> None:
|
||||
data, resp = process_image_buffer(
|
||||
path.read_bytes(), maxsize=512, quality=60, maxzoom=2.0
|
||||
)
|
||||
_assert_ok(data, resp, backend="pyvips")
|
||||
_assert_ok(data, resp, backend="vips")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -127,7 +127,7 @@ def test_process_pdf() -> None:
|
||||
maxzoom=2.0,
|
||||
quality=60,
|
||||
)
|
||||
_assert_ok(data, resp, backend="pdf+pyvips")
|
||||
_assert_ok(data, resp, backend="pdf+vips")
|
||||
assert resp.width == 595
|
||||
assert resp.height == 842
|
||||
|
||||
@@ -138,9 +138,9 @@ def test_process_pdf() -> None:
|
||||
|
||||
|
||||
DISPATCH_FIXTURES = [
|
||||
("Landscape_1.jpg", "pyvips", 1800, 1200),
|
||||
("Landscape_1.jpg", "vips", 1800, 1200),
|
||||
("sample-1mb.mp4", "video", 854, 480),
|
||||
("sample.pdf", "pdf+pyvips", 595, 842),
|
||||
("sample.pdf", "pdf+vips", 595, 842),
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user