onlyoffice previews much faster and more robust, added --oosetup helper, improved error/log handling
This commit is contained in:
+26
-20
@@ -5,7 +5,7 @@ from pathlib import Path
|
|||||||
from docopt import docopt
|
from docopt import docopt
|
||||||
|
|
||||||
import cista
|
import cista
|
||||||
from cista import app, config, droppy, serve, server80
|
from cista import app, config, droppy, onlyoffice, serve, server80
|
||||||
from cista.util import pwgen
|
from cista.util import pwgen
|
||||||
|
|
||||||
del app, server80.app # Only import needed, for Sanic multiprocessing
|
del app, server80.app # Only import needed, for Sanic multiprocessing
|
||||||
@@ -53,34 +53,33 @@ def create_startup_box(
|
|||||||
|
|
||||||
banner = create_banner()
|
banner = create_banner()
|
||||||
|
|
||||||
doc = """\
|
_default_confdir = (
|
||||||
|
(Path(os.environ["XDG_CONFIG_HOME"]) / "cista").as_posix()
|
||||||
|
if os.environ.get("XDG_CONFIG_HOME")
|
||||||
|
else (Path.home() / ".config/cista").as_posix()
|
||||||
|
)
|
||||||
|
|
||||||
|
doc = f"""\
|
||||||
Usage:
|
Usage:
|
||||||
cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
|
cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
|
||||||
cista [-c <confdir>] --user <name> [--privileged] [--password]
|
cista [-c <confdir>] --user <name> [--privileged] [--password]
|
||||||
|
cista [-c <confdir>] --oosetup
|
||||||
cista --version
|
cista --version
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-c CONFDIR Custom config directory
|
-c CONFDIR Config directory [{_default_confdir}]
|
||||||
-l, --listen LISTEN-ADDR
|
-l, --listen ADDR Listen on address (port, :port, /socket or domain for https)
|
||||||
Listen on
|
|
||||||
:8989 (localhost port, plain http)
|
|
||||||
<addr>:3000 (bind another address, port)
|
|
||||||
/path/to/unix.sock (unix socket)
|
|
||||||
example.com (run on 80 and 443 with LetsEncrypt)
|
|
||||||
--import-droppy Import Droppy config from ~/.droppy/config
|
--import-droppy Import Droppy config from ~/.droppy/config
|
||||||
--dev Developer mode (reloads, friendlier crashes, more logs)
|
--dev Developer mode (reloads, friendlier crashes, more logs)
|
||||||
|
--user NAME Create or modify a user account (when server is not running)
|
||||||
Listen address and path are preserved in config,
|
--privileged Grant admin rights
|
||||||
and only config dir and dev mode need to be specified on subsequent runs.
|
|
||||||
|
|
||||||
User management:
|
|
||||||
--user NAME Create or modify user
|
|
||||||
--privileged Give the user full admin rights
|
|
||||||
--password Reset password
|
--password Reset password
|
||||||
|
--oosetup Build and run OnlyOffice in Docker for document previews
|
||||||
|
|
||||||
Environment:
|
Environment:
|
||||||
PASKIA_BACKEND_URL Paskia single sign-on (e.g. http://localhost:4401)
|
PASKIA_BACKEND_URL Paskia single sign-on (e.g. http://localhost:4401)
|
||||||
https://git.zi.fi/leovasanko/paskia
|
https://git.zi.fi/leovasanko/paskia
|
||||||
|
ONLYOFFICE_CISTA_URL, ONLYOFFICE_JWT_SECRET, ONLYOFFICE_CALLBACK_HOST (if needed)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
first_time_help = """\
|
first_time_help = """\
|
||||||
@@ -115,6 +114,8 @@ def _main():
|
|||||||
args = docopt(doc)
|
args = docopt(doc)
|
||||||
if args["--user"]:
|
if args["--user"]:
|
||||||
return _user(args)
|
return _user(args)
|
||||||
|
if args["--oosetup"]:
|
||||||
|
return onlyoffice.setup_docker(_resolve_confdir(args))
|
||||||
listen = args["--listen"]
|
listen = args["--listen"]
|
||||||
# Validate arguments first
|
# Validate arguments first
|
||||||
if args["<path>"]:
|
if args["<path>"]:
|
||||||
@@ -171,17 +172,22 @@ def _main():
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _confdir(args):
|
def _resolve_confdir(args):
|
||||||
|
confdir = None
|
||||||
if args["-c"]:
|
if args["-c"]:
|
||||||
# Custom config directory
|
# Custom config directory
|
||||||
confdir = Path(args["-c"]).resolve()
|
confdir = Path(args["-c"]).resolve()
|
||||||
if confdir.exists() and not confdir.is_dir():
|
if confdir.exists() and not confdir.is_dir():
|
||||||
if confdir.name != config.conffile.name:
|
if confdir.name != "db.toml":
|
||||||
raise ValueError("Config path is not a directory")
|
raise ValueError("Config path is not a directory")
|
||||||
# Accidentally pointed to the db.toml, use parent
|
# Accidentally pointed to the db.toml, use parent
|
||||||
confdir = confdir.parent
|
confdir = confdir.parent
|
||||||
os.environ["CISTA_HOME"] = confdir.as_posix()
|
return confdir
|
||||||
config.init_confdir() # Uses environ if available
|
|
||||||
|
|
||||||
|
def _confdir(args):
|
||||||
|
confdir = _resolve_confdir(args)
|
||||||
|
config.init_confdir(confdir)
|
||||||
|
|
||||||
|
|
||||||
def _user(args):
|
def _user(args):
|
||||||
|
|||||||
+16
-1
@@ -16,7 +16,17 @@ from setproctitle import setproctitle
|
|||||||
from stream_zip import ZIP_AUTO, stream_zip
|
from stream_zip import ZIP_AUTO, stream_zip
|
||||||
from zstandard import ZstdCompressor
|
from zstandard import ZstdCompressor
|
||||||
|
|
||||||
from cista import auth, config, fileserver, onlyoffice, preview, session, sharefs, sso, watching
|
from cista import (
|
||||||
|
auth,
|
||||||
|
config,
|
||||||
|
fileserver,
|
||||||
|
onlyoffice,
|
||||||
|
preview,
|
||||||
|
session,
|
||||||
|
sharefs,
|
||||||
|
sso,
|
||||||
|
watching,
|
||||||
|
)
|
||||||
from cista.api import bp
|
from cista.api import bp
|
||||||
from cista.preview import shutdown_preview_workers, start_preview_workers
|
from cista.preview import shutdown_preview_workers, start_preview_workers
|
||||||
from cista.sanic_logging import (
|
from cista.sanic_logging import (
|
||||||
@@ -126,6 +136,11 @@ async def main_start(app):
|
|||||||
watching.start(app)
|
watching.start(app)
|
||||||
|
|
||||||
|
|
||||||
|
@app.after_server_start
|
||||||
|
async def main_after_start(app):
|
||||||
|
onlyoffice.log_reachable_info()
|
||||||
|
|
||||||
|
|
||||||
# Sanic sometimes fails to execute after_server_stop, so we do it before instead (potentially interrupting handlers)
|
# Sanic sometimes fails to execute after_server_stop, so we do it before instead (potentially interrupting handlers)
|
||||||
@app.before_server_stop
|
@app.before_server_stop
|
||||||
async def main_stop(app):
|
async def main_stop(app):
|
||||||
|
|||||||
+3
-3
@@ -61,10 +61,10 @@ config: Config
|
|||||||
conffile: Path
|
conffile: Path
|
||||||
|
|
||||||
|
|
||||||
def init_confdir() -> None:
|
def init_confdir(confdir: Path | str | None = None) -> None:
|
||||||
global conffile
|
global conffile
|
||||||
if p := os.environ.get("CISTA_HOME"):
|
if confdir is not None:
|
||||||
home = Path(p)
|
home = Path(confdir).expanduser()
|
||||||
else:
|
else:
|
||||||
xdg = os.environ.get("XDG_CONFIG_HOME")
|
xdg = os.environ.get("XDG_CONFIG_HOME")
|
||||||
home = (
|
home = (
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Patched OnlyOffice Document Server with configurable converter worker count.
|
||||||
|
#
|
||||||
|
# The Community Edition hardcodes the document converter to 1 worker,
|
||||||
|
# which creates a severe bottleneck under concurrent load.
|
||||||
|
# This image patches the open-source license.js to spawn a configurable
|
||||||
|
# number of converter workers (default 8).
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build -t onlyoffice-cista docker/onlyoffice-converter-patch
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker run -d -p 8988:80 \
|
||||||
|
# -e WORKERS=16 \
|
||||||
|
# -e JWT_SECRET=your-strong-secret \
|
||||||
|
# --name onlyoffice onlyoffice-cista
|
||||||
|
#
|
||||||
|
# JWT:
|
||||||
|
# Set JWT_SECRET to the same value you pass to Cista as ONLYOFFICE_JWT_SECRET.
|
||||||
|
# OnlyOffice will enable token validation automatically.
|
||||||
|
#
|
||||||
|
# The ONLYOFFICE_VERSION build arg lets you target a specific release.
|
||||||
|
|
||||||
|
ARG ONLYOFFICE_VERSION=9.3.1
|
||||||
|
|
||||||
|
FROM onlyoffice/documentserver:${ONLYOFFICE_VERSION}
|
||||||
|
|
||||||
|
# Prevent interactive apt prompts
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# Install Node.js, npm, and git so we can run the FileConverter from source.
|
||||||
|
RUN apt-get update -qq && \
|
||||||
|
apt-get install -y -qq --no-install-recommends \
|
||||||
|
nodejs \
|
||||||
|
npm \
|
||||||
|
git \
|
||||||
|
ca-certificates && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Clone the open-source server components (shallow, ~15 MB).
|
||||||
|
# The master branch is used because the Linux/web tags are not published
|
||||||
|
# in the server repo; the license.js file has been stable for years.
|
||||||
|
RUN git clone --depth 1 https://github.com/ONLYOFFICE/server.git /opt/oo-server
|
||||||
|
|
||||||
|
# Patch license.js so the converter worker count is read from an env var
|
||||||
|
# instead of being hardcoded to 1.
|
||||||
|
RUN sed -i \
|
||||||
|
's/count: 1,/count: parseInt(process.env.WORKERS, 10) || 8,/' \
|
||||||
|
/opt/oo-server/Common/sources/license.js
|
||||||
|
|
||||||
|
# Install npm dependencies for the modules the FileConverter touches.
|
||||||
|
# DocService deps are also needed because converter.js pulls in baseConnector.
|
||||||
|
RUN cd /opt/oo-server/Common && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/FileConverter && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/DocService && npm ci --no-audit --no-fund
|
||||||
|
|
||||||
|
# Back up the compiled pkg binary and replace it with our wrapper.
|
||||||
|
RUN mv /var/www/onlyoffice/documentserver/server/FileConverter/converter \
|
||||||
|
/var/www/onlyoffice/documentserver/server/FileConverter/converter.orig
|
||||||
|
|
||||||
|
COPY converter-wrapper.sh /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
RUN chmod +x /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
|
||||||
|
# Default worker count (override at runtime with -e WORKERS=16).
|
||||||
|
ENV WORKERS=8
|
||||||
|
|
||||||
|
# Use our custom entrypoint to persist the env var to a file that the
|
||||||
|
# non-root converter process (user=ds) can read.
|
||||||
|
COPY entrypoint.sh /app/ds/run-document-server-patched.sh
|
||||||
|
RUN chmod +x /app/ds/run-document-server-patched.sh
|
||||||
|
ENTRYPOINT ["/app/ds/run-document-server-patched.sh"]
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Wrapper that runs the OnlyOffice FileConverter from patched Node.js source.
|
||||||
|
# Replaces the compiled pkg binary shipped with the Community Edition.
|
||||||
|
|
||||||
|
# The env var is not passed through supervisor to the 'ds' user, so we read
|
||||||
|
# it from a file written by the custom entrypoint.
|
||||||
|
if [ -z "${WORKERS}" ] && [ -r /tmp/oo-converter-workers.txt ]; then
|
||||||
|
export WORKERS=$(cat /tmp/oo-converter-workers.txt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd /opt/oo-server/FileConverter || exit 1
|
||||||
|
|
||||||
|
export NODE_ENV=production-linux
|
||||||
|
export NODE_CONFIG_DIR=/etc/onlyoffice/documentserver
|
||||||
|
export NODE_DISABLE_COLORS=1
|
||||||
|
export APPLICATION_NAME=onlyoffice
|
||||||
|
export LD_LIBRARY_PATH=/var/www/onlyoffice/documentserver/server/FileConverter/bin
|
||||||
|
|
||||||
|
exec node sources/convertermaster.js "$@"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Custom entrypoint that persists WORKERS to a file readable by
|
||||||
|
# the non-root user that supervisor uses to run the converter.
|
||||||
|
|
||||||
|
echo "${WORKERS:-8}" > /tmp/oo-converter-workers.txt
|
||||||
|
chmod 644 /tmp/oo-converter-workers.txt
|
||||||
|
|
||||||
|
exec /app/ds/run-document-server.sh "$@"
|
||||||
+85
-71
@@ -17,6 +17,7 @@ import socket
|
|||||||
import socketserver
|
import socketserver
|
||||||
import subprocess
|
import subprocess
|
||||||
import threading
|
import threading
|
||||||
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from http.server import SimpleHTTPRequestHandler
|
from http.server import SimpleHTTPRequestHandler
|
||||||
@@ -28,6 +29,8 @@ import httpx
|
|||||||
import jwt
|
import jwt
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
|
|
||||||
|
from cista import config
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Configuration helpers
|
# Configuration helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -37,11 +40,14 @@ _httpx_client: httpx.AsyncClient | None = None
|
|||||||
|
|
||||||
|
|
||||||
def _get_onlyoffice_url() -> str:
|
def _get_onlyoffice_url() -> str:
|
||||||
return os.environ.get("ONLYOFFICE_URL", "http://localhost:8988")
|
return os.environ.get("ONLYOFFICE_CISTA_URL", "http://localhost:8988")
|
||||||
|
|
||||||
|
|
||||||
def _get_jwt_secret() -> str | None:
|
def _get_jwt_secret() -> str:
|
||||||
return os.environ.get("ONLYOFFICE_JWT_SECRET") or None
|
return (
|
||||||
|
os.environ.get("ONLYOFFICE_JWT_SECRET")
|
||||||
|
or config.derived_secret("onlyoffice", size=16).hex()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_callback_host() -> str:
|
def _get_callback_host() -> str:
|
||||||
@@ -93,14 +99,85 @@ async def close_oo_client() -> None:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def is_available() -> bool:
|
def _probe_status() -> tuple[bool, bool, str | None]:
|
||||||
"""Return True if the configured OnlyOffice Document Server is reachable."""
|
"""Return (ok, responded, detail) for a lightweight reachability probe."""
|
||||||
url = _get_onlyoffice_url().rstrip("/") + "/ConvertService.ashx"
|
url = _get_onlyoffice_url().rstrip("/") + "/ConvertService.ashx"
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=3) as resp: # noqa: S310
|
with urllib.request.urlopen(url, timeout=2) as resp: # noqa: S310
|
||||||
return resp.status in (200, 405) # 405 Method Not Allowed is fine, means endpoint exists
|
status = resp.status
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
status = e.code
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False, False, None
|
||||||
|
|
||||||
|
if status in (200, 405):
|
||||||
|
return True, True, None
|
||||||
|
if status >= 500:
|
||||||
|
return False, True, f"HTTP {status}"
|
||||||
|
return False, True, f"HTTP {status}"
|
||||||
|
|
||||||
|
|
||||||
|
def log_reachable_info() -> None:
|
||||||
|
"""Log info on success, warning on responded probe errors, silent on no-response."""
|
||||||
|
ok, responded, detail = _probe_status()
|
||||||
|
if ok:
|
||||||
|
logger.info("Using OnlyOffice document server at %s", _get_onlyoffice_url())
|
||||||
|
elif responded:
|
||||||
|
suffix = f": {detail}" if detail else ""
|
||||||
|
logger.warning("OnlyOffice probe failed%s", suffix)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_docker(confdir: Path | None = None) -> int:
|
||||||
|
"""Build and run the patched OnlyOffice Docker image."""
|
||||||
|
config.init_confdir(confdir)
|
||||||
|
if config.conffile.exists():
|
||||||
|
config.load_config()
|
||||||
|
else:
|
||||||
|
config.update_config(
|
||||||
|
{
|
||||||
|
"listen": ":8989",
|
||||||
|
"path": Path.home() / "Downloads",
|
||||||
|
"public": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
secret = config.derived_secret("onlyoffice", size=16).hex()
|
||||||
|
docker_dir = Path(__file__).parent / "docker"
|
||||||
|
if not docker_dir.is_dir():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Docker files not found at {docker_dir}. Is the package installed correctly?"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Building OnlyOffice image")
|
||||||
|
build_cmd = ["docker", "build", "-t", "onlyoffice-cista", str(docker_dir)]
|
||||||
|
logger.info("%s", " ".join(build_cmd))
|
||||||
|
result = subprocess.run(build_cmd)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("Failed to build OnlyOffice image")
|
||||||
|
|
||||||
|
logger.info("Starting OnlyOffice container")
|
||||||
|
run_cmd = [
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"-p",
|
||||||
|
"8988:80",
|
||||||
|
"-e",
|
||||||
|
f"JWT_SECRET={secret}",
|
||||||
|
"-e",
|
||||||
|
"WORKERS=8",
|
||||||
|
"--name",
|
||||||
|
"onlyoffice-cista",
|
||||||
|
"--restart",
|
||||||
|
"unless-stopped",
|
||||||
|
"onlyoffice-cista",
|
||||||
|
]
|
||||||
|
logger.info("%s", " ".join(run_cmd))
|
||||||
|
result = subprocess.run(run_cmd)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("Failed to start OnlyOffice container")
|
||||||
|
logger.info("OnlyOffice is running on http://localhost:8988")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
async def is_available_async(timeout: float = 2.0) -> bool:
|
async def is_available_async(timeout: float = 2.0) -> bool:
|
||||||
@@ -175,69 +252,6 @@ def _build_jwt_token(payload: dict) -> str | None:
|
|||||||
return jwt.encode(payload, secret, algorithm="HS256")
|
return jwt.encode(payload, secret, algorithm="HS256")
|
||||||
|
|
||||||
|
|
||||||
def convert_to_png(file_path: Path, timeout: float = 5.0) -> bytes:
|
|
||||||
"""Convert *file_path* to PNG using OnlyOffice Document Server.
|
|
||||||
|
|
||||||
Returns the PNG bytes. Raises RuntimeError on failure.
|
|
||||||
"""
|
|
||||||
oo_url = _get_onlyoffice_url().rstrip("/")
|
|
||||||
convert_url = f"{oo_url}/ConvertService.ashx"
|
|
||||||
|
|
||||||
# Start temporary HTTP server so OnlyOffice can fetch the file
|
|
||||||
doc_url, httpd = _serve_file_temporarily(file_path)
|
|
||||||
try:
|
|
||||||
suffix = file_path.suffix.lstrip(".").lower()
|
|
||||||
payload = {
|
|
||||||
"async": False,
|
|
||||||
"filetype": suffix,
|
|
||||||
"key": f"cista_{file_path.stat().st_mtime_ns}",
|
|
||||||
"outputtype": "png",
|
|
||||||
"title": file_path.name,
|
|
||||||
"url": doc_url,
|
|
||||||
}
|
|
||||||
|
|
||||||
headers = {"Content-Type": "application/json"}
|
|
||||||
token = _build_jwt_token(payload)
|
|
||||||
if token:
|
|
||||||
# Conversion API expects JWT in request body when token checks are enabled.
|
|
||||||
payload["token"] = token
|
|
||||||
headers["Authorization"] = token
|
|
||||||
|
|
||||||
req = urllib.request.Request( # noqa: S310
|
|
||||||
convert_url,
|
|
||||||
data=json.dumps(payload).encode(),
|
|
||||||
headers=headers,
|
|
||||||
method="POST",
|
|
||||||
)
|
|
||||||
|
|
||||||
t_start = perf_counter()
|
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
|
|
||||||
body = resp.read()
|
|
||||||
t_end = perf_counter()
|
|
||||||
|
|
||||||
# Parse XML response
|
|
||||||
text = body.decode("utf-8", errors="replace")
|
|
||||||
if "<Error>" in text:
|
|
||||||
code = "unknown"
|
|
||||||
if "<Error>" in text and "</Error>" in text:
|
|
||||||
code = text.split("<Error>")[1].split("</Error>")[0]
|
|
||||||
raise RuntimeError(f"OnlyOffice conversion error: {code}")
|
|
||||||
|
|
||||||
if "<FileUrl>" not in text:
|
|
||||||
raise RuntimeError("OnlyOffice response did not contain FileUrl")
|
|
||||||
|
|
||||||
file_url = text.split("<FileUrl>")[1].split("</FileUrl>")[0]
|
|
||||||
file_url = file_url.replace("&", "&")
|
|
||||||
|
|
||||||
logger.debug("OnlyOffice converted in %.2fs: %s", t_end - t_start, file_url)
|
|
||||||
|
|
||||||
# Download converted PNG
|
|
||||||
with urllib.request.urlopen(file_url, timeout=timeout) as png_resp: # noqa: S310
|
|
||||||
return png_resp.read()
|
|
||||||
finally:
|
|
||||||
httpd.shutdown()
|
|
||||||
|
|
||||||
|
|
||||||
async def convert_to_png_async(file_path: Path, timeout: float = 5.0) -> bytes:
|
async def convert_to_png_async(file_path: Path, timeout: float = 5.0) -> bytes:
|
||||||
"""Convert *file_path* to PNG using OnlyOffice Document Server (async).
|
"""Convert *file_path* to PNG using OnlyOffice Document Server (async).
|
||||||
|
|
||||||
|
|||||||
+47
-48
@@ -19,6 +19,7 @@ from wsgiref.handlers import format_date_time
|
|||||||
|
|
||||||
import av
|
import av
|
||||||
import fitz # PyMuPDF
|
import fitz # PyMuPDF
|
||||||
|
import httpx
|
||||||
import msgspec
|
import msgspec
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pyvips
|
import pyvips
|
||||||
@@ -100,7 +101,12 @@ class _PreviewWorker:
|
|||||||
self.proc = proc
|
self.proc = proc
|
||||||
|
|
||||||
async def request(
|
async def request(
|
||||||
self, filepath, quality: int, maxsize: int, maxzoom: float, data: bytes | None = None
|
self,
|
||||||
|
filepath,
|
||||||
|
quality: int,
|
||||||
|
maxsize: int,
|
||||||
|
maxzoom: float,
|
||||||
|
data: bytes | None = None,
|
||||||
):
|
):
|
||||||
if self.proc.returncode is not None:
|
if self.proc.returncode is not None:
|
||||||
raise WorkerProtocolError("worker already exited")
|
raise WorkerProtocolError("worker already exited")
|
||||||
@@ -172,9 +178,7 @@ class _PreviewWorkerPool:
|
|||||||
)
|
)
|
||||||
_active_procs.add(proc)
|
_active_procs.add(proc)
|
||||||
try:
|
try:
|
||||||
ready = await asyncio.wait_for(
|
ready = await asyncio.wait_for(proc.stdout.readexactly(1), timeout=30.0)
|
||||||
proc.stdout.readexactly(1), timeout=30.0
|
|
||||||
)
|
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
with contextlib.suppress(ProcessLookupError):
|
with contextlib.suppress(ProcessLookupError):
|
||||||
proc.kill()
|
proc.kill()
|
||||||
@@ -184,9 +188,7 @@ class _PreviewWorkerPool:
|
|||||||
"preview worker exited before signalling readiness"
|
"preview worker exited before signalling readiness"
|
||||||
)
|
)
|
||||||
if ready != b"\x01":
|
if ready != b"\x01":
|
||||||
raise WorkerProtocolError(
|
raise WorkerProtocolError(f"preview worker ready signal invalid: {ready!r}")
|
||||||
f"preview worker ready signal invalid: {ready!r}"
|
|
||||||
)
|
|
||||||
return _PreviewWorker(proc)
|
return _PreviewWorker(proc)
|
||||||
|
|
||||||
async def _add_worker(self) -> None:
|
async def _add_worker(self) -> None:
|
||||||
@@ -280,9 +282,7 @@ class _PreviewWorkerPool:
|
|||||||
)
|
)
|
||||||
if not future.done():
|
if not future.done():
|
||||||
future.set_exception(
|
future.set_exception(
|
||||||
PreviewError(
|
PreviewError(f"unexpected worker error for {filepath.name}")
|
||||||
f"unexpected worker error for {filepath.name}"
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
if replace:
|
if replace:
|
||||||
@@ -303,7 +303,12 @@ class _PreviewWorkerPool:
|
|||||||
self._dispatchers.append(asyncio.create_task(self._dispatch_loop()))
|
self._dispatchers.append(asyncio.create_task(self._dispatch_loop()))
|
||||||
|
|
||||||
async def run(
|
async def run(
|
||||||
self, filepath, quality: int, maxsize: int, maxzoom: float, data: bytes | None = None
|
self,
|
||||||
|
filepath,
|
||||||
|
quality: int,
|
||||||
|
maxsize: int,
|
||||||
|
maxzoom: float,
|
||||||
|
data: bytes | None = None,
|
||||||
):
|
):
|
||||||
if self._closed:
|
if self._closed:
|
||||||
raise PreviewError("preview worker pool closed")
|
raise PreviewError("preview worker pool closed")
|
||||||
@@ -389,10 +394,6 @@ class PreviewTimeoutError(Exception):
|
|||||||
"""Raised when the preview subprocess exceeds PREVIEW_TIMEOUT."""
|
"""Raised when the preview subprocess exceeds PREVIEW_TIMEOUT."""
|
||||||
|
|
||||||
|
|
||||||
class OnlyOfficeUnavailableError(Exception):
|
|
||||||
"""Raised when the OnlyOffice Document Server is not reachable."""
|
|
||||||
|
|
||||||
|
|
||||||
class PreviewError(Exception):
|
class PreviewError(Exception):
|
||||||
"""Raised when the preview subprocess exits with a non-zero status."""
|
"""Raised when the preview subprocess exits with a non-zero status."""
|
||||||
|
|
||||||
@@ -410,7 +411,7 @@ class PreviewError(Exception):
|
|||||||
|
|
||||||
# Max concurrent OnlyOffice conversion requests. OO has its own queue;
|
# Max concurrent OnlyOffice conversion requests. OO has its own queue;
|
||||||
# we must not flood it. This is intentionally small.
|
# we must not flood it. This is intentionally small.
|
||||||
OO_MAX_CONCURRENT = 2
|
OO_MAX_CONCURRENT = PREVIEW_WORKERS
|
||||||
|
|
||||||
|
|
||||||
class OOConversionManager:
|
class OOConversionManager:
|
||||||
@@ -467,9 +468,6 @@ async def _generate_office_preview(
|
|||||||
filepath: Path, quality: int, maxsize: int, maxzoom: float
|
filepath: Path, quality: int, maxsize: int, maxzoom: float
|
||||||
) -> tuple[bytes | None, PreviewResponse | None]:
|
) -> tuple[bytes | None, PreviewResponse | None]:
|
||||||
"""Generate a preview for an office file using OnlyOffice + worker AVIF conversion."""
|
"""Generate a preview for an office file using OnlyOffice + worker AVIF conversion."""
|
||||||
if not await onlyoffice.is_available_async():
|
|
||||||
raise OnlyOfficeUnavailableError("OnlyOffice Document Server is not reachable")
|
|
||||||
|
|
||||||
manager = get_oo_manager()
|
manager = get_oo_manager()
|
||||||
t_oo_start = perf_counter()
|
t_oo_start = perf_counter()
|
||||||
png_bytes = await manager.convert(filepath)
|
png_bytes = await manager.convert(filepath)
|
||||||
@@ -540,6 +538,20 @@ OFFICE_PREVIEW_SUFFIXES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _onlyoffice_error_short_text(detail: str) -> str:
|
||||||
|
if detail.startswith("OnlyOffice conversion error:"):
|
||||||
|
code = detail.rsplit(":", 1)[-1].strip()
|
||||||
|
return {
|
||||||
|
"-8": "onlyoffice jwt error",
|
||||||
|
"-4": "onlyoffice input error",
|
||||||
|
"-2": "onlyoffice timeout error",
|
||||||
|
"-1": "onlyoffice unknown error",
|
||||||
|
}.get(code, f"onlyoffice {code} error")
|
||||||
|
if "OnlyOffice response did not contain FileUrl" in detail:
|
||||||
|
return "onlyoffice no-fileurl error"
|
||||||
|
return "onlyoffice error"
|
||||||
|
|
||||||
|
|
||||||
def _preview_job_priority(path) -> int:
|
def _preview_job_priority(path) -> int:
|
||||||
"""Return priority for preview job (lower=higher priority).
|
"""Return priority for preview job (lower=higher priority).
|
||||||
|
|
||||||
@@ -624,9 +636,18 @@ async def preview(req, path):
|
|||||||
except PreviewTimeoutError:
|
except PreviewTimeoutError:
|
||||||
logger.warning("Preview worker timeout for %s", filepath)
|
logger.warning("Preview worker timeout for %s", filepath)
|
||||||
return empty(503)
|
return empty(503)
|
||||||
except OnlyOfficeUnavailableError:
|
except httpx.HTTPStatusError as e:
|
||||||
req.ctx._log_extra = "onlyoffice N/A"
|
req.ctx._log_extra = "onlyoffice N/A"
|
||||||
return empty(503)
|
return empty(503)
|
||||||
|
except httpx.RequestError:
|
||||||
|
req.ctx._log_extra = "onlyoffice N/A"
|
||||||
|
return empty(503)
|
||||||
|
except RuntimeError as e:
|
||||||
|
detail = str(e)
|
||||||
|
if detail.startswith("OnlyOffice"):
|
||||||
|
req.ctx._log_extra = _onlyoffice_error_short_text(detail)
|
||||||
|
return empty(503)
|
||||||
|
raise
|
||||||
except PreviewError as e:
|
except PreviewError as e:
|
||||||
if e.backend:
|
if e.backend:
|
||||||
req.ctx._log_extra = e.backend
|
req.ctx._log_extra = e.backend
|
||||||
@@ -637,6 +658,9 @@ async def preview(req, path):
|
|||||||
detail = captured.splitlines()[0]
|
detail = captured.splitlines()[0]
|
||||||
logger.error("%s preview: %s", filepath, detail)
|
logger.error("%s preview: %s", filepath, detail)
|
||||||
return empty(422)
|
return empty(422)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
req.ctx._log_extra = "preview cancelled"
|
||||||
|
return empty(503)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Unhandled preview error for %s", filepath)
|
logger.exception("Unhandled preview error for %s", filepath)
|
||||||
return empty(500)
|
return empty(500)
|
||||||
@@ -677,7 +701,9 @@ def dispatch(path, quality, maxsize, maxzoom, data=None):
|
|||||||
try:
|
try:
|
||||||
if data:
|
if data:
|
||||||
backend = "pyvips"
|
backend = "pyvips"
|
||||||
return process_image_buffer(data, quality=quality, maxsize=maxsize, maxzoom=maxzoom)
|
return process_image_buffer(
|
||||||
|
data, quality=quality, maxsize=maxsize, maxzoom=maxzoom
|
||||||
|
)
|
||||||
suffix = path.suffix.lower()
|
suffix = path.suffix.lower()
|
||||||
if suffix in DOC_PREVIEW_SUFFIXES:
|
if suffix in DOC_PREVIEW_SUFFIXES:
|
||||||
backend = "pdf"
|
backend = "pdf"
|
||||||
@@ -852,33 +878,6 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def process_office(path, *, quality, maxsize, maxzoom):
|
|
||||||
t_load_start = perf_counter()
|
|
||||||
if not onlyoffice.is_available():
|
|
||||||
raise RuntimeError("OnlyOffice Document Server is not reachable")
|
|
||||||
png_bytes = onlyoffice.convert_to_png(path)
|
|
||||||
t_load_end = perf_counter()
|
|
||||||
|
|
||||||
t_save_start = perf_counter()
|
|
||||||
img = pyvips.Image.new_from_buffer(png_bytes, "")
|
|
||||||
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
|
|
||||||
if scale < 1.0:
|
|
||||||
img = img.resize(scale)
|
|
||||||
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True)
|
|
||||||
backend = "onlyoffice+pyvips"
|
|
||||||
t_save_end = perf_counter()
|
|
||||||
|
|
||||||
return ret, PreviewResponse(
|
|
||||||
ok=True,
|
|
||||||
mime="image/avif",
|
|
||||||
backend=backend,
|
|
||||||
timings=[
|
|
||||||
round((t_load_end - t_load_start) * 1000, 1),
|
|
||||||
round((t_save_end - t_save_start) * 1000, 1),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def process_video(path, *, maxsize, quality):
|
def process_video(path, *, maxsize, quality):
|
||||||
frame = None
|
frame = None
|
||||||
imgdata = io.BytesIO()
|
imgdata = io.BytesIO()
|
||||||
|
|||||||
@@ -134,12 +134,24 @@ def _run_loop() -> None:
|
|||||||
def main() -> None:
|
def main() -> None:
|
||||||
# Configure all log output to stderr before any imports that may emit logs.
|
# Configure all log output to stderr before any imports that may emit logs.
|
||||||
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
|
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
|
||||||
|
try:
|
||||||
|
from cista import config
|
||||||
|
|
||||||
|
config.load_config()
|
||||||
|
logging.warning(
|
||||||
|
"preview-worker config=%s master_secret=%s",
|
||||||
|
config.conffile,
|
||||||
|
config.config.secret,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logging.exception("preview-worker failed to load config at startup")
|
||||||
if len(sys.argv) > 1:
|
if len(sys.argv) > 1:
|
||||||
_run_once()
|
_run_once()
|
||||||
return
|
return
|
||||||
# Eagerly import heavy modules before signalling readiness so the parent
|
# Eagerly import heavy modules before signalling readiness so the parent
|
||||||
# does not hand us a request while we are still initialising.
|
# does not hand us a request while we are still initialising.
|
||||||
from cista.preview import dispatch # noqa: F401
|
from cista.preview import dispatch # noqa: F401
|
||||||
|
|
||||||
sys.stdout.buffer.write(b"\x01")
|
sys.stdout.buffer.write(b"\x01")
|
||||||
sys.stdout.buffer.flush()
|
sys.stdout.buffer.flush()
|
||||||
_run_loop()
|
_run_loop()
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
services:
|
||||||
|
onlyoffice:
|
||||||
|
build:
|
||||||
|
context: ./docker/onlyoffice-converter-patch
|
||||||
|
args:
|
||||||
|
ONLYOFFICE_VERSION: "9.3.1"
|
||||||
|
container_name: onlyoffice
|
||||||
|
ports:
|
||||||
|
- "8080:80"
|
||||||
|
environment:
|
||||||
|
# Number of converter workers (default 8).
|
||||||
|
# Set to your CPU count or slightly below.
|
||||||
|
- WORKERS
|
||||||
|
# JWT secret shared with Cista.
|
||||||
|
# OnlyOffice reads it as JWT_SECRET; Cista reads it as ONLYOFFICE_JWT_SECRET.
|
||||||
|
# We use ONLYOFFICE_JWT_SECRET as the canonical name so you only set one variable.
|
||||||
|
- JWT_SECRET=${ONLYOFFICE_JWT_SECRET}
|
||||||
|
- JWT_ENABLED=true
|
||||||
|
- JWT_HEADER=Authorization
|
||||||
|
volumes:
|
||||||
|
# Persist fonts and generated caches across restarts
|
||||||
|
- onlyoffice-data:/var/www/onlyoffice/Data
|
||||||
|
- onlyoffice-lib:/var/lib/onlyoffice
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
onlyoffice-data:
|
||||||
|
onlyoffice-lib:
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Patched OnlyOffice Document Server with configurable converter worker count.
|
||||||
|
#
|
||||||
|
# The Community Edition hardcodes the document converter to 1 worker,
|
||||||
|
# which creates a severe bottleneck under concurrent load.
|
||||||
|
# This image patches the open-source license.js to spawn a configurable
|
||||||
|
# number of converter workers (default 8).
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build -t onlyoffice-cista docker/onlyoffice-converter-patch
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker run -d -p 8988:80 \
|
||||||
|
# -e WORKERS=16 \
|
||||||
|
# -e JWT_SECRET=your-strong-secret \
|
||||||
|
# --name onlyoffice onlyoffice-cista
|
||||||
|
#
|
||||||
|
# JWT:
|
||||||
|
# Set JWT_SECRET to the same value you pass to Cista as ONLYOFFICE_JWT_SECRET.
|
||||||
|
# OnlyOffice will enable token validation automatically.
|
||||||
|
#
|
||||||
|
# The ONLYOFFICE_VERSION build arg lets you target a specific release.
|
||||||
|
|
||||||
|
ARG ONLYOFFICE_VERSION=9.3.1
|
||||||
|
|
||||||
|
FROM onlyoffice/documentserver:${ONLYOFFICE_VERSION}
|
||||||
|
|
||||||
|
# Prevent interactive apt prompts
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# Install Node.js, npm, and git so we can run the FileConverter from source.
|
||||||
|
RUN apt-get update -qq && \
|
||||||
|
apt-get install -y -qq --no-install-recommends \
|
||||||
|
nodejs \
|
||||||
|
npm \
|
||||||
|
git \
|
||||||
|
ca-certificates && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Clone the open-source server components (shallow, ~15 MB).
|
||||||
|
# The master branch is used because the Linux/web tags are not published
|
||||||
|
# in the server repo; the license.js file has been stable for years.
|
||||||
|
RUN git clone --depth 1 https://github.com/ONLYOFFICE/server.git /opt/oo-server
|
||||||
|
|
||||||
|
# Patch license.js so the converter worker count is read from an env var
|
||||||
|
# instead of being hardcoded to 1.
|
||||||
|
RUN sed -i \
|
||||||
|
's/count: 1,/count: parseInt(process.env.WORKERS, 10) || 8,/' \
|
||||||
|
/opt/oo-server/Common/sources/license.js
|
||||||
|
|
||||||
|
# Install npm dependencies for the modules the FileConverter touches.
|
||||||
|
# DocService deps are also needed because converter.js pulls in baseConnector.
|
||||||
|
RUN cd /opt/oo-server/Common && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/FileConverter && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/DocService && npm ci --no-audit --no-fund
|
||||||
|
|
||||||
|
# Back up the compiled pkg binary and replace it with our wrapper.
|
||||||
|
RUN mv /var/www/onlyoffice/documentserver/server/FileConverter/converter \
|
||||||
|
/var/www/onlyoffice/documentserver/server/FileConverter/converter.orig
|
||||||
|
|
||||||
|
COPY converter-wrapper.sh /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
RUN chmod +x /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
|
||||||
|
# Default worker count (override at runtime with -e WORKERS=16).
|
||||||
|
ENV WORKERS=8
|
||||||
|
|
||||||
|
# Use our custom entrypoint to persist the env var to a file that the
|
||||||
|
# non-root converter process (user=ds) can read.
|
||||||
|
COPY entrypoint.sh /app/ds/run-document-server-patched.sh
|
||||||
|
RUN chmod +x /app/ds/run-document-server-patched.sh
|
||||||
|
ENTRYPOINT ["/app/ds/run-document-server-patched.sh"]
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Wrapper that runs the OnlyOffice FileConverter from patched Node.js source.
|
||||||
|
# Replaces the compiled pkg binary shipped with the Community Edition.
|
||||||
|
|
||||||
|
# The env var is not passed through supervisor to the 'ds' user, so we read
|
||||||
|
# it from a file written by the custom entrypoint.
|
||||||
|
if [ -z "${WORKERS}" ] && [ -r /tmp/oo-converter-workers.txt ]; then
|
||||||
|
export WORKERS=$(cat /tmp/oo-converter-workers.txt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd /opt/oo-server/FileConverter || exit 1
|
||||||
|
|
||||||
|
export NODE_ENV=production-linux
|
||||||
|
export NODE_CONFIG_DIR=/etc/onlyoffice/documentserver
|
||||||
|
export NODE_DISABLE_COLORS=1
|
||||||
|
export APPLICATION_NAME=onlyoffice
|
||||||
|
export LD_LIBRARY_PATH=/var/www/onlyoffice/documentserver/server/FileConverter/bin
|
||||||
|
|
||||||
|
exec node sources/convertermaster.js "$@"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Custom entrypoint that persists WORKERS to a file readable by
|
||||||
|
# the non-root user that supervisor uses to run the converter.
|
||||||
|
|
||||||
|
echo "${WORKERS:-8}" > /tmp/oo-converter-workers.txt
|
||||||
|
chmod 644 /tmp/oo-converter-workers.txt
|
||||||
|
|
||||||
|
exec /app/ds/run-document-server.sh "$@"
|
||||||
+1
-1
@@ -77,7 +77,7 @@ docs = [
|
|||||||
source = "vcs"
|
source = "vcs"
|
||||||
|
|
||||||
[tool.hatch.build]
|
[tool.hatch.build]
|
||||||
artifacts = ["cista/frontend-build"]
|
artifacts = ["cista/frontend-build", "cista/docker"]
|
||||||
targets.sdist.hooks.custom.path = "scripts/fastapi-vue/build-frontend.py"
|
targets.sdist.hooks.custom.path = "scripts/fastapi-vue/build-frontend.py"
|
||||||
targets.sdist.include = [
|
targets.sdist.include = [
|
||||||
"/cista",
|
"/cista",
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -27,8 +26,7 @@ def _persist_config():
|
|||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def setup_storage(tmp_path: Path):
|
def setup_storage(tmp_path: Path):
|
||||||
os.environ["CISTA_HOME"] = str(tmp_path)
|
config.init_confdir(tmp_path)
|
||||||
config.init_confdir()
|
|
||||||
user = config.User()
|
user = config.User()
|
||||||
auth.set_password(user, "secret")
|
auth.set_password(user, "secret")
|
||||||
admin = config.User(privileged=True)
|
admin = config.User(privileged=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user