Vasanko/cista-storage
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

onlyoffice previews much faster and more robust, added --oosetup helper, improved error/log handling

Browse Source
This commit is contained in:
LeoVasanko
2026-05-02 03:10:17 +00:00
parent 4dd1d4c7e6
commit 1bec73f4cd
15 changed files with 420 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cista/__main__.py
+33 -27
View File
@@ -5,7 +5,7 @@ from pathlib import Path
from docopt import docopt
import cista
from cista import app, config, droppy, serve, server80
from cista import app, config, droppy, onlyoffice, serve, server80
from cista.util import pwgen
del app, server80.app # Only import needed, for Sanic multiprocessing
@@ -53,40 +53,39 @@ def create_startup_box(
banner = create_banner()
doc = """\
_default_confdir = (
(Path(os.environ["XDG_CONFIG_HOME"]) / "cista").as_posix()
if os.environ.get("XDG_CONFIG_HOME")
else (Path.home() / ".config/cista").as_posix()
)
doc = f"""\
Usage:
cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
cista [-c <confdir>] --user <name> [--privileged] [--password]
cista [-c <confdir>] --oosetup
cista --version
Options:
-c CONFDIR Custom config directory
-l, --listen LISTEN-ADDR
Listen on
:8989 (localhost port, plain http)
<addr>:3000 (bind another address, port)
/path/to/unix.sock (unix socket)
example.com (run on 80 and 443 with LetsEncrypt)
--import-droppy Import Droppy config from ~/.droppy/config
--dev Developer mode (reloads, friendlier crashes, more logs)
Listen address and path are preserved in config,
and only config dir and dev mode need to be specified on subsequent runs.
User management:
--user NAME Create or modify user
--privileged Give the user full admin rights
--password Reset password
-c CONFDIR Config directory [{_default_confdir}]
-l, --listen ADDR Listen on address (port, :port, /socket or domain for https)
--import-droppy Import Droppy config from ~/.droppy/config
--dev Developer mode (reloads, friendlier crashes, more logs)
--user NAME Create or modify a user account (when server is not running)
--privileged Grant admin rights
--password Reset password
--oosetup Build and run OnlyOffice in Docker for document previews
Environment:
PASKIA_BACKEND_URL Paskia single sign-on (e.g. http://localhost:4401)
https://git.zi.fi/leovasanko/paskia
PASKIA_BACKEND_URL Paskia single sign-on (e.g. http://localhost:4401)
https://git.zi.fi/leovasanko/paskia
ONLYOFFICE_CISTA_URL, ONLYOFFICE_JWT_SECRET, ONLYOFFICE_CALLBACK_HOST (if needed)
"""
first_time_help = """\
No config file found! Get started with:
cista --user yourname --privileged # If you want user accounts
cista -l :8989 /path/to/files # Run the server on localhost:8989
cista --user yourname --privileged # If you want user accounts
cista -l :8989 /path/to/files # Run the server on localhost:8989
See cista --help for other options!
"""
@@ -115,6 +114,8 @@ def _main():
args = docopt(doc)
if args["--user"]:
return _user(args)
if args["--oosetup"]:
return onlyoffice.setup_docker(_resolve_confdir(args))
listen = args["--listen"]
# Validate arguments first
if args["<path>"]:
@@ -171,17 +172,22 @@ def _main():
return 0
def _confdir(args):
def _resolve_confdir(args):
confdir = None
if args["-c"]:
# Custom config directory
confdir = Path(args["-c"]).resolve()
if confdir.exists() and not confdir.is_dir():
if confdir.name != config.conffile.name:
if confdir.name != "db.toml":
raise ValueError("Config path is not a directory")
# Accidentally pointed to the db.toml, use parent
confdir = confdir.parent
os.environ["CISTA_HOME"] = confdir.as_posix()
config.init_confdir() # Uses environ if available
return confdir
def _confdir(args):
confdir = _resolve_confdir(args)
config.init_confdir(confdir)
def _user(args):
cista/app.py
+16 -1
View File
@@ -16,7 +16,17 @@ from setproctitle import setproctitle
from stream_zip import ZIP_AUTO, stream_zip
from zstandard import ZstdCompressor
from cista import auth, config, fileserver, onlyoffice, preview, session, sharefs, sso, watching
from cista import (
auth,
config,
fileserver,
onlyoffice,
preview,
session,
sharefs,
sso,
watching,
)
from cista.api import bp
from cista.preview import shutdown_preview_workers, start_preview_workers
from cista.sanic_logging import (
@@ -126,6 +136,11 @@ async def main_start(app):
watching.start(app)
@app.after_server_start
async def main_after_start(app):
onlyoffice.log_reachable_info()
# Sanic sometimes fails to execute after_server_stop, so we do it before instead (potentially interrupting handlers)
@app.before_server_stop
async def main_stop(app):
cista/config.py
+3 -3
View File
@@ -61,10 +61,10 @@ config: Config
conffile: Path
def init_confdir() -> None:
def init_confdir(confdir: Path | str | None = None) -> None:
global conffile
if p := os.environ.get("CISTA_HOME"):
home = Path(p)
if confdir is not None:
home = Path(confdir).expanduser()
else:
xdg = os.environ.get("XDG_CONFIG_HOME")
home = (
cista/docker/Dockerfile
+70
View File
@@ -0,0 +1,70 @@
# Patched OnlyOffice Document Server with configurable converter worker count.
#
# The Community Edition hardcodes the document converter to 1 worker,
# which creates a severe bottleneck under concurrent load.
# This image patches the open-source license.js to spawn a configurable
# number of converter workers (default 8).
#
# Build:
# docker build -t onlyoffice-cista docker/onlyoffice-converter-patch
#
# Run:
# docker run -d -p 8988:80 \
# -e WORKERS=16 \
# -e JWT_SECRET=your-strong-secret \
# --name onlyoffice onlyoffice-cista
#
# JWT:
# Set JWT_SECRET to the same value you pass to Cista as ONLYOFFICE_JWT_SECRET.
# OnlyOffice will enable token validation automatically.
#
# The ONLYOFFICE_VERSION build arg lets you target a specific release.
ARG ONLYOFFICE_VERSION=9.3.1
FROM onlyoffice/documentserver:${ONLYOFFICE_VERSION}
# Prevent interactive apt prompts
ENV DEBIAN_FRONTEND=noninteractive
# Install Node.js, npm, and git so we can run the FileConverter from source.
RUN apt-get update -qq && \
apt-get install -y -qq --no-install-recommends \
nodejs \
npm \
git \
ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Clone the open-source server components (shallow, ~15 MB).
# The master branch is used because the Linux/web tags are not published
# in the server repo; the license.js file has been stable for years.
RUN git clone --depth 1 https://github.com/ONLYOFFICE/server.git /opt/oo-server
# Patch license.js so the converter worker count is read from an env var
# instead of being hardcoded to 1.
RUN sed -i \
's/count: 1,/count: parseInt(process.env.WORKERS, 10) || 8,/' \
/opt/oo-server/Common/sources/license.js
# Install npm dependencies for the modules the FileConverter touches.
# DocService deps are also needed because converter.js pulls in baseConnector.
RUN cd /opt/oo-server/Common && npm ci --no-audit --no-fund
RUN cd /opt/oo-server/FileConverter && npm ci --no-audit --no-fund
RUN cd /opt/oo-server/DocService && npm ci --no-audit --no-fund
# Back up the compiled pkg binary and replace it with our wrapper.
RUN mv /var/www/onlyoffice/documentserver/server/FileConverter/converter \
/var/www/onlyoffice/documentserver/server/FileConverter/converter.orig
COPY converter-wrapper.sh /var/www/onlyoffice/documentserver/server/FileConverter/converter
RUN chmod +x /var/www/onlyoffice/documentserver/server/FileConverter/converter
# Default worker count (override at runtime with -e WORKERS=16).
ENV WORKERS=8
# Use our custom entrypoint to persist the env var to a file that the
# non-root converter process (user=ds) can read.
COPY entrypoint.sh /app/ds/run-document-server-patched.sh
RUN chmod +x /app/ds/run-document-server-patched.sh
ENTRYPOINT ["/app/ds/run-document-server-patched.sh"]
cista/docker/converter-wrapper.sh
+19
View File
@@ -0,0 +1,19 @@
#!/bin/bash
# Wrapper that runs the OnlyOffice FileConverter from patched Node.js source.
# Replaces the compiled pkg binary shipped with the Community Edition.
# The env var is not passed through supervisor to the 'ds' user, so we read
# it from a file written by the custom entrypoint.
if [ -z "${WORKERS}" ] && [ -r /tmp/oo-converter-workers.txt ]; then
export WORKERS=$(cat /tmp/oo-converter-workers.txt)
fi
cd /opt/oo-server/FileConverter || exit 1
export NODE_ENV=production-linux
export NODE_CONFIG_DIR=/etc/onlyoffice/documentserver
export NODE_DISABLE_COLORS=1
export APPLICATION_NAME=onlyoffice
export LD_LIBRARY_PATH=/var/www/onlyoffice/documentserver/server/FileConverter/bin
exec node sources/convertermaster.js "$@"
cista/docker/entrypoint.sh
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
# Custom entrypoint that persists WORKERS to a file readable by
# the non-root user that supervisor uses to run the converter.
echo "${WORKERS:-8}" > /tmp/oo-converter-workers.txt
chmod 644 /tmp/oo-converter-workers.txt
exec /app/ds/run-document-server.sh "$@"
cista/onlyoffice.py
+85 -71
View File
@@ -17,6 +17,7 @@ import socket
import socketserver
import subprocess
import threading
import urllib.error
import urllib.request
from functools import partial
from http.server import SimpleHTTPRequestHandler
@@ -28,6 +29,8 @@ import httpx
import jwt
from sanic.log import logger
from cista import config
# ---------------------------------------------------------------------------
# Configuration helpers
# ---------------------------------------------------------------------------
@@ -37,11 +40,14 @@ _httpx_client: httpx.AsyncClient | None = None
def _get_onlyoffice_url() -> str:
return os.environ.get("ONLYOFFICE_URL", "http://localhost:8988")
return os.environ.get("ONLYOFFICE_CISTA_URL", "http://localhost:8988")
def _get_jwt_secret() -> str | None:
return os.environ.get("ONLYOFFICE_JWT_SECRET") or None
def _get_jwt_secret() -> str:
return (
os.environ.get("ONLYOFFICE_JWT_SECRET")
or config.derived_secret("onlyoffice", size=16).hex()
)
def _get_callback_host() -> str:
@@ -93,14 +99,85 @@ async def close_oo_client() -> None:
# ---------------------------------------------------------------------------
def is_available() -> bool:
"""Return True if the configured OnlyOffice Document Server is reachable."""
def _probe_status() -> tuple[bool, bool, str | None]:
"""Return (ok, responded, detail) for a lightweight reachability probe."""
url = _get_onlyoffice_url().rstrip("/") + "/ConvertService.ashx"
try:
with urllib.request.urlopen(url, timeout=3) as resp: # noqa: S310
return resp.status in (200, 405) # 405 Method Not Allowed is fine, means endpoint exists
with urllib.request.urlopen(url, timeout=2) as resp: # noqa: S310
status = resp.status
except urllib.error.HTTPError as e:
status = e.code
except Exception:
return False
return False, False, None
if status in (200, 405):
return True, True, None
if status >= 500:
return False, True, f"HTTP {status}"
return False, True, f"HTTP {status}"
def log_reachable_info() -> None:
"""Log info on success, warning on responded probe errors, silent on no-response."""
ok, responded, detail = _probe_status()
if ok:
logger.info("Using OnlyOffice document server at %s", _get_onlyoffice_url())
elif responded:
suffix = f": {detail}" if detail else ""
logger.warning("OnlyOffice probe failed%s", suffix)
def setup_docker(confdir: Path | None = None) -> int:
"""Build and run the patched OnlyOffice Docker image."""
config.init_confdir(confdir)
if config.conffile.exists():
config.load_config()
else:
config.update_config(
{
"listen": ":8989",
"path": Path.home() / "Downloads",
"public": False,
}
)
secret = config.derived_secret("onlyoffice", size=16).hex()
docker_dir = Path(__file__).parent / "docker"
if not docker_dir.is_dir():
raise FileNotFoundError(
f"Docker files not found at {docker_dir}. Is the package installed correctly?"
)
logger.info("Building OnlyOffice image")
build_cmd = ["docker", "build", "-t", "onlyoffice-cista", str(docker_dir)]
logger.info("%s", " ".join(build_cmd))
result = subprocess.run(build_cmd)
if result.returncode != 0:
raise RuntimeError("Failed to build OnlyOffice image")
logger.info("Starting OnlyOffice container")
run_cmd = [
"docker",
"run",
"-d",
"-p",
"8988:80",
"-e",
f"JWT_SECRET={secret}",
"-e",
"WORKERS=8",
"--name",
"onlyoffice-cista",
"--restart",
"unless-stopped",
"onlyoffice-cista",
]
logger.info("%s", " ".join(run_cmd))
result = subprocess.run(run_cmd)
if result.returncode != 0:
raise RuntimeError("Failed to start OnlyOffice container")
logger.info("OnlyOffice is running on http://localhost:8988")
return 0
async def is_available_async(timeout: float = 2.0) -> bool:
@@ -175,69 +252,6 @@ def _build_jwt_token(payload: dict) -> str | None:
return jwt.encode(payload, secret, algorithm="HS256")
def convert_to_png(file_path: Path, timeout: float = 5.0) -> bytes:
"""Convert *file_path* to PNG using OnlyOffice Document Server.
Returns the PNG bytes. Raises RuntimeError on failure.
"""
oo_url = _get_onlyoffice_url().rstrip("/")
convert_url = f"{oo_url}/ConvertService.ashx"
# Start temporary HTTP server so OnlyOffice can fetch the file
doc_url, httpd = _serve_file_temporarily(file_path)
try:
suffix = file_path.suffix.lstrip(".").lower()
payload = {
"async": False,
"filetype": suffix,
"key": f"cista_{file_path.stat().st_mtime_ns}",
"outputtype": "png",
"title": file_path.name,
"url": doc_url,
}
headers = {"Content-Type": "application/json"}
token = _build_jwt_token(payload)
if token:
# Conversion API expects JWT in request body when token checks are enabled.
payload["token"] = token
headers["Authorization"] = token
req = urllib.request.Request( # noqa: S310
convert_url,
data=json.dumps(payload).encode(),
headers=headers,
method="POST",
)
t_start = perf_counter()
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
body = resp.read()
t_end = perf_counter()
# Parse XML response
text = body.decode("utf-8", errors="replace")
if "<Error>" in text:
code = "unknown"
if "<Error>" in text and "</Error>" in text:
code = text.split("<Error>")[1].split("</Error>")[0]
raise RuntimeError(f"OnlyOffice conversion error: {code}")
if "<FileUrl>" not in text:
raise RuntimeError("OnlyOffice response did not contain FileUrl")
file_url = text.split("<FileUrl>")[1].split("</FileUrl>")[0]
file_url = file_url.replace("&amp;", "&")
logger.debug("OnlyOffice converted in %.2fs: %s", t_end - t_start, file_url)
# Download converted PNG
with urllib.request.urlopen(file_url, timeout=timeout) as png_resp: # noqa: S310
return png_resp.read()
finally:
httpd.shutdown()
async def convert_to_png_async(file_path: Path, timeout: float = 5.0) -> bytes:
"""Convert *file_path* to PNG using OnlyOffice Document Server (async).
cista/preview.py
+47 -48
View File
@@ -19,6 +19,7 @@ from wsgiref.handlers import format_date_time
import av
import fitz # PyMuPDF
import httpx
import msgspec
import numpy as np
import pyvips
@@ -100,7 +101,12 @@ class _PreviewWorker:
self.proc = proc
async def request(
self, filepath, quality: int, maxsize: int, maxzoom: float, data: bytes | None = None
self,
filepath,
quality: int,
maxsize: int,
maxzoom: float,
data: bytes | None = None,
):
if self.proc.returncode is not None:
raise WorkerProtocolError("worker already exited")
@@ -172,9 +178,7 @@ class _PreviewWorkerPool:
)
_active_procs.add(proc)
try:
ready = await asyncio.wait_for(
proc.stdout.readexactly(1), timeout=30.0
)
ready = await asyncio.wait_for(proc.stdout.readexactly(1), timeout=30.0)
except asyncio.TimeoutError:
with contextlib.suppress(ProcessLookupError):
proc.kill()
@@ -184,9 +188,7 @@ class _PreviewWorkerPool:
"preview worker exited before signalling readiness"
)
if ready != b"\x01":
raise WorkerProtocolError(
f"preview worker ready signal invalid: {ready!r}"
)
raise WorkerProtocolError(f"preview worker ready signal invalid: {ready!r}")
return _PreviewWorker(proc)
async def _add_worker(self) -> None:
@@ -280,9 +282,7 @@ class _PreviewWorkerPool:
)
if not future.done():
future.set_exception(
PreviewError(
f"unexpected worker error for {filepath.name}"
)
PreviewError(f"unexpected worker error for {filepath.name}")
)
finally:
if replace:
@@ -303,7 +303,12 @@ class _PreviewWorkerPool:
self._dispatchers.append(asyncio.create_task(self._dispatch_loop()))
async def run(
self, filepath, quality: int, maxsize: int, maxzoom: float, data: bytes | None = None
self,
filepath,
quality: int,
maxsize: int,
maxzoom: float,
data: bytes | None = None,
):
if self._closed:
raise PreviewError("preview worker pool closed")
@@ -389,10 +394,6 @@ class PreviewTimeoutError(Exception):
"""Raised when the preview subprocess exceeds PREVIEW_TIMEOUT."""
class OnlyOfficeUnavailableError(Exception):
"""Raised when the OnlyOffice Document Server is not reachable."""
class PreviewError(Exception):
"""Raised when the preview subprocess exits with a non-zero status."""
@@ -410,7 +411,7 @@ class PreviewError(Exception):
# Max concurrent OnlyOffice conversion requests. OO has its own queue;
# we must not flood it. This is intentionally small.
OO_MAX_CONCURRENT = 2
OO_MAX_CONCURRENT = PREVIEW_WORKERS
class OOConversionManager:
@@ -467,9 +468,6 @@ async def _generate_office_preview(
filepath: Path, quality: int, maxsize: int, maxzoom: float
) -> tuple[bytes | None, PreviewResponse | None]:
"""Generate a preview for an office file using OnlyOffice + worker AVIF conversion."""
if not await onlyoffice.is_available_async():
raise OnlyOfficeUnavailableError("OnlyOffice Document Server is not reachable")
manager = get_oo_manager()
t_oo_start = perf_counter()
png_bytes = await manager.convert(filepath)
@@ -540,6 +538,20 @@ OFFICE_PREVIEW_SUFFIXES = {
}
def _onlyoffice_error_short_text(detail: str) -> str:
if detail.startswith("OnlyOffice conversion error:"):
code = detail.rsplit(":", 1)[-1].strip()
return {
"-8": "onlyoffice jwt error",
"-4": "onlyoffice input error",
"-2": "onlyoffice timeout error",
"-1": "onlyoffice unknown error",
}.get(code, f"onlyoffice {code} error")
if "OnlyOffice response did not contain FileUrl" in detail:
return "onlyoffice no-fileurl error"
return "onlyoffice error"
def _preview_job_priority(path) -> int:
"""Return priority for preview job (lower=higher priority).
@@ -624,9 +636,18 @@ async def preview(req, path):
except PreviewTimeoutError:
logger.warning("Preview worker timeout for %s", filepath)
return empty(503)
except OnlyOfficeUnavailableError:
except httpx.HTTPStatusError as e:
req.ctx._log_extra = "onlyoffice N/A"
return empty(503)
except httpx.RequestError:
req.ctx._log_extra = "onlyoffice N/A"
return empty(503)
except RuntimeError as e:
detail = str(e)
if detail.startswith("OnlyOffice"):
req.ctx._log_extra = _onlyoffice_error_short_text(detail)
return empty(503)
raise
except PreviewError as e:
if e.backend:
req.ctx._log_extra = e.backend
@@ -637,6 +658,9 @@ async def preview(req, path):
detail = captured.splitlines()[0]
logger.error("%s preview: %s", filepath, detail)
return empty(422)
except asyncio.CancelledError:
req.ctx._log_extra = "preview cancelled"
return empty(503)
except Exception:
logger.exception("Unhandled preview error for %s", filepath)
return empty(500)
@@ -677,7 +701,9 @@ def dispatch(path, quality, maxsize, maxzoom, data=None):
try:
if data:
backend = "pyvips"
return process_image_buffer(data, quality=quality, maxsize=maxsize, maxzoom=maxzoom)
return process_image_buffer(
data, quality=quality, maxsize=maxsize, maxzoom=maxzoom
)
suffix = path.suffix.lower()
if suffix in DOC_PREVIEW_SUFFIXES:
backend = "pdf"
@@ -852,33 +878,6 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
)
def process_office(path, *, quality, maxsize, maxzoom):
t_load_start = perf_counter()
if not onlyoffice.is_available():
raise RuntimeError("OnlyOffice Document Server is not reachable")
png_bytes = onlyoffice.convert_to_png(path)
t_load_end = perf_counter()
t_save_start = perf_counter()
img = pyvips.Image.new_from_buffer(png_bytes, "")
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
if scale < 1.0:
img = img.resize(scale)
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True)
backend = "onlyoffice+pyvips"
t_save_end = perf_counter()
return ret, PreviewResponse(
ok=True,
mime="image/avif",
backend=backend,
timings=[
round((t_load_end - t_load_start) * 1000, 1),
round((t_save_end - t_save_start) * 1000, 1),
],
)
def process_video(path, *, maxsize, quality):
frame = None
imgdata = io.BytesIO()
cista/preview_worker.py
+12
View File
@@ -134,12 +134,24 @@ def _run_loop() -> None:
def main() -> None:
# Configure all log output to stderr before any imports that may emit logs.
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
try:
from cista import config
config.load_config()
logging.warning(
"preview-worker config=%s master_secret=%s",
config.conffile,
config.config.secret,
)
except Exception:
logging.exception("preview-worker failed to load config at startup")
if len(sys.argv) > 1:
_run_once()
return
# Eagerly import heavy modules before signalling readiness so the parent
# does not hand us a request while we are still initialising.
from cista.preview import dispatch # noqa: F401
sys.stdout.buffer.write(b"\x01")
sys.stdout.buffer.flush()
_run_loop()