Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af804e2c9f | ||
|
|
1dc0c4441a | ||
|
|
c7ba0d5a04 | ||
|
|
0071058b29 | ||
|
|
338c74de69 | ||
|
|
b13f08eab2 | ||
|
|
5d566d6deb | ||
|
|
5c4965a86b | ||
|
|
bd7291e9ef | ||
|
|
2fa52229cc | ||
|
|
2dea459d8f | ||
|
|
922069c603 | ||
|
|
5c7c7343ad | ||
|
|
593d16d8c5 | ||
|
|
20d8d317fa | ||
|
|
8d89c397a4 | ||
|
|
1bec73f4cd | ||
|
|
4dd1d4c7e6 | ||
|
|
84ef91a360 | ||
|
|
421d90e9c5 | ||
|
|
8b4e622aef | ||
|
|
13f32c57ab | ||
|
|
25a2a5f20c | ||
|
|
041090cce9 |
+29
-23
@@ -5,7 +5,8 @@ from pathlib import Path
|
|||||||
from docopt import docopt
|
from docopt import docopt
|
||||||
|
|
||||||
import cista
|
import cista
|
||||||
from cista import app, config, droppy, serve, server80
|
from cista import app, config, droppy, onlyoffice, serve, server80
|
||||||
|
from cista.sso import PASKIA_BACKEND_URL
|
||||||
from cista.util import pwgen
|
from cista.util import pwgen
|
||||||
|
|
||||||
del app, server80.app # Only import needed, for Sanic multiprocessing
|
del app, server80.app # Only import needed, for Sanic multiprocessing
|
||||||
@@ -30,7 +31,7 @@ def create_startup_box(
|
|||||||
):
|
):
|
||||||
"""Create a framed startup box with server information."""
|
"""Create a framed startup box with server information."""
|
||||||
title = f"Cista {cista.__version__}"
|
title = f"Cista {cista.__version__}"
|
||||||
listen = unix if unix else url
|
listen = unix or url
|
||||||
location = f"{folder} @ {listen}"
|
location = f"{folder} @ {listen}"
|
||||||
lines = [title, location]
|
lines = [title, location]
|
||||||
# Auth line: Paskia <url> or Password, with optional Public suffix
|
# Auth line: Paskia <url> or Password, with optional Public suffix
|
||||||
@@ -53,34 +54,33 @@ def create_startup_box(
|
|||||||
|
|
||||||
banner = create_banner()
|
banner = create_banner()
|
||||||
|
|
||||||
doc = """\
|
_default_confdir = (
|
||||||
|
(Path(os.environ["XDG_CONFIG_HOME"]) / "cista").as_posix()
|
||||||
|
if os.environ.get("XDG_CONFIG_HOME")
|
||||||
|
else (Path.home() / ".config/cista").as_posix()
|
||||||
|
)
|
||||||
|
|
||||||
|
doc = f"""\
|
||||||
Usage:
|
Usage:
|
||||||
cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
|
cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
|
||||||
cista [-c <confdir>] --user <name> [--privileged] [--password]
|
cista [-c <confdir>] --user <name> [--privileged] [--password]
|
||||||
|
cista [-c <confdir>] --oosetup
|
||||||
cista --version
|
cista --version
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-c CONFDIR Custom config directory
|
-c CONFDIR Config directory [{_default_confdir}]
|
||||||
-l, --listen LISTEN-ADDR
|
-l, --listen ADDR Listen on address (port, :port, /socket or domain for https)
|
||||||
Listen on
|
|
||||||
:8989 (localhost port, plain http)
|
|
||||||
<addr>:3000 (bind another address, port)
|
|
||||||
/path/to/unix.sock (unix socket)
|
|
||||||
example.com (run on 80 and 443 with LetsEncrypt)
|
|
||||||
--import-droppy Import Droppy config from ~/.droppy/config
|
--import-droppy Import Droppy config from ~/.droppy/config
|
||||||
--dev Developer mode (reloads, friendlier crashes, more logs)
|
--dev Developer mode (reloads, friendlier crashes, more logs)
|
||||||
|
--user NAME Create or modify a user account (when server is not running)
|
||||||
Listen address and path are preserved in config,
|
--privileged Grant admin rights
|
||||||
and only config dir and dev mode need to be specified on subsequent runs.
|
|
||||||
|
|
||||||
User management:
|
|
||||||
--user NAME Create or modify user
|
|
||||||
--privileged Give the user full admin rights
|
|
||||||
--password Reset password
|
--password Reset password
|
||||||
|
--oosetup Build and run OnlyOffice in Docker for document previews
|
||||||
|
|
||||||
Environment:
|
Environment:
|
||||||
PASKIA_BACKEND_URL Paskia single sign-on (e.g. http://localhost:4401)
|
PASKIA_BACKEND_URL Paskia single sign-on (e.g. http://localhost:4401)
|
||||||
https://git.zi.fi/leovasanko/paskia
|
https://git.zi.fi/leovasanko/paskia
|
||||||
|
ONLYOFFICE_CISTA_URL, ONLYOFFICE_JWT_SECRET, ONLYOFFICE_CALLBACK_HOST (if needed)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
first_time_help = """\
|
first_time_help = """\
|
||||||
@@ -115,6 +115,8 @@ def _main():
|
|||||||
args = docopt(doc)
|
args = docopt(doc)
|
||||||
if args["--user"]:
|
if args["--user"]:
|
||||||
return _user(args)
|
return _user(args)
|
||||||
|
if args["--oosetup"]:
|
||||||
|
return onlyoffice.setup_docker(_resolve_confdir(args))
|
||||||
listen = args["--listen"]
|
listen = args["--listen"]
|
||||||
# Validate arguments first
|
# Validate arguments first
|
||||||
if args["<path>"]:
|
if args["<path>"]:
|
||||||
@@ -153,9 +155,6 @@ def _main():
|
|||||||
if not config.config.path.is_dir():
|
if not config.config.path.is_dir():
|
||||||
raise ValueError(f"No such directory: {config.config.path}")
|
raise ValueError(f"No such directory: {config.config.path}")
|
||||||
dev = args["--dev"]
|
dev = args["--dev"]
|
||||||
# Check for Paskia SSO
|
|
||||||
from cista.sso import PASKIA_BACKEND_URL
|
|
||||||
|
|
||||||
# Print startup box
|
# Print startup box
|
||||||
startup_box = create_startup_box(
|
startup_box = create_startup_box(
|
||||||
folder=config.config.path,
|
folder=config.config.path,
|
||||||
@@ -171,17 +170,24 @@ def _main():
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _confdir(args):
|
def _resolve_confdir(args):
|
||||||
|
confdir = None
|
||||||
if args["-c"]:
|
if args["-c"]:
|
||||||
# Custom config directory
|
# Custom config directory
|
||||||
confdir = Path(args["-c"]).resolve()
|
confdir = Path(args["-c"]).resolve()
|
||||||
if confdir.exists() and not confdir.is_dir():
|
if confdir.exists() and not confdir.is_dir():
|
||||||
if confdir.name != config.conffile.name:
|
if confdir.name != "db.toml":
|
||||||
raise ValueError("Config path is not a directory")
|
raise ValueError("Config path is not a directory")
|
||||||
# Accidentally pointed to the db.toml, use parent
|
# Accidentally pointed to the db.toml, use parent
|
||||||
confdir = confdir.parent
|
confdir = confdir.parent
|
||||||
|
return confdir
|
||||||
|
|
||||||
|
|
||||||
|
def _confdir(args):
|
||||||
|
confdir = _resolve_confdir(args)
|
||||||
|
if confdir is not None:
|
||||||
os.environ["CISTA_HOME"] = confdir.as_posix()
|
os.environ["CISTA_HOME"] = confdir.as_posix()
|
||||||
config.init_confdir() # Uses environ if available
|
config.init_confdir()
|
||||||
|
|
||||||
|
|
||||||
def _user(args):
|
def _user(args):
|
||||||
|
|||||||
+9
-9
@@ -6,7 +6,7 @@ from sanic import Blueprint, json
|
|||||||
from sanic.exceptions import BadRequest
|
from sanic.exceptions import BadRequest
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
|
|
||||||
from cista import __version__, auth, config, sharefs, sso, watching
|
from cista import __version__, auth, config, onlyoffice, sharefs, sso, watching
|
||||||
from cista.auth import (
|
from cista.auth import (
|
||||||
create_share_token_handler,
|
create_share_token_handler,
|
||||||
create_token_handler,
|
create_token_handler,
|
||||||
@@ -22,11 +22,13 @@ fileserver = FileServer()
|
|||||||
|
|
||||||
@bp.before_server_start
|
@bp.before_server_start
|
||||||
async def start_fileserver(app):
|
async def start_fileserver(app):
|
||||||
|
_ = app
|
||||||
await fileserver.start()
|
await fileserver.start()
|
||||||
|
|
||||||
|
|
||||||
@bp.after_server_stop
|
@bp.after_server_stop
|
||||||
async def stop_fileserver(app):
|
async def stop_fileserver(app):
|
||||||
|
_ = app
|
||||||
await fileserver.stop()
|
await fileserver.stop()
|
||||||
|
|
||||||
|
|
||||||
@@ -63,6 +65,7 @@ async def watch(req, ws):
|
|||||||
"version": __version__,
|
"version": __version__,
|
||||||
"public": config.config.public,
|
"public": config.config.public,
|
||||||
"paskia": sso.paskia_enabled(),
|
"paskia": sso.paskia_enabled(),
|
||||||
|
"office_previews": await onlyoffice.is_available_cached(),
|
||||||
},
|
},
|
||||||
"user": user_info,
|
"user": user_info,
|
||||||
}
|
}
|
||||||
@@ -99,6 +102,7 @@ async def watch(req, ws):
|
|||||||
|
|
||||||
|
|
||||||
def subscribe(uuid, ws):
|
def subscribe(uuid, ws):
|
||||||
|
_ = ws
|
||||||
with watching.state.lock:
|
with watching.state.lock:
|
||||||
q = watching.pubsub[uuid] = asyncio.Queue()
|
q = watching.pubsub[uuid] = asyncio.Queue()
|
||||||
# Init with disk usage and full tree
|
# Init with disk usage and full tree
|
||||||
@@ -125,12 +129,10 @@ async def update_public(request):
|
|||||||
await auth.verify(request, privileged=True)
|
await auth.verify(request, privileged=True)
|
||||||
try:
|
try:
|
||||||
public = request.json["public"]
|
public = request.json["public"]
|
||||||
if not isinstance(public, bool):
|
|
||||||
raise ValueError("public must be a boolean")
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise BadRequest("Missing public field") from None
|
raise BadRequest("Missing public field") from None
|
||||||
except ValueError as e:
|
if not isinstance(public, bool):
|
||||||
raise BadRequest(str(e)) from None
|
raise BadRequest("public must be a boolean")
|
||||||
config.update_config({"public": public})
|
config.update_config({"public": public})
|
||||||
return json({"message": "Public access setting updated", "public": public})
|
return json({"message": "Public access setting updated", "public": public})
|
||||||
|
|
||||||
@@ -140,12 +142,10 @@ async def update_name(request):
|
|||||||
await auth.verify(request, privileged=True)
|
await auth.verify(request, privileged=True)
|
||||||
try:
|
try:
|
||||||
name = request.json["name"]
|
name = request.json["name"]
|
||||||
if not isinstance(name, str):
|
|
||||||
raise ValueError("name must be a string")
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise BadRequest("Missing name field") from None
|
raise BadRequest("Missing name field") from None
|
||||||
except ValueError as e:
|
if not isinstance(name, str):
|
||||||
raise BadRequest(str(e)) from None
|
raise BadRequest("name must be a string")
|
||||||
config.update_config({"name": name})
|
config.update_config({"name": name})
|
||||||
# Return the effective name (fallback to path.name if empty)
|
# Return the effective name (fallback to path.name if empty)
|
||||||
effective_name = name or config.config.path.name
|
effective_name = name or config.config.path.name
|
||||||
|
|||||||
+36
-11
@@ -8,6 +8,7 @@ from stat import S_IFDIR, S_IFREG
|
|||||||
from urllib.parse import unquote
|
from urllib.parse import unquote
|
||||||
from wsgiref.handlers import format_date_time
|
from wsgiref.handlers import format_date_time
|
||||||
|
|
||||||
|
import tracerite
|
||||||
from blake3 import blake3
|
from blake3 import blake3
|
||||||
from sanic import Sanic, empty, raw, redirect
|
from sanic import Sanic, empty, raw, redirect
|
||||||
from sanic.exceptions import Forbidden, NotFound
|
from sanic.exceptions import Forbidden, NotFound
|
||||||
@@ -16,7 +17,17 @@ from setproctitle import setproctitle
|
|||||||
from stream_zip import ZIP_AUTO, stream_zip
|
from stream_zip import ZIP_AUTO, stream_zip
|
||||||
from zstandard import ZstdCompressor
|
from zstandard import ZstdCompressor
|
||||||
|
|
||||||
from cista import auth, config, fileserver, preview, session, sharefs, sso, watching
|
from cista import (
|
||||||
|
auth,
|
||||||
|
config,
|
||||||
|
fileserver,
|
||||||
|
onlyoffice,
|
||||||
|
preview,
|
||||||
|
session,
|
||||||
|
sharefs,
|
||||||
|
sso,
|
||||||
|
watching,
|
||||||
|
)
|
||||||
from cista.api import bp
|
from cista.api import bp
|
||||||
from cista.preview import shutdown_preview_workers, start_preview_workers
|
from cista.preview import shutdown_preview_workers, start_preview_workers
|
||||||
from cista.sanic_logging import (
|
from cista.sanic_logging import (
|
||||||
@@ -27,8 +38,10 @@ from cista.sanic_logging import (
|
|||||||
from cista.sanic_logging import logger as access_logger
|
from cista.sanic_logging import logger as access_logger
|
||||||
from cista.util.apphelpers import handle_sanic_exception
|
from cista.util.apphelpers import handle_sanic_exception
|
||||||
|
|
||||||
|
tracerite.load()
|
||||||
configure_access_logging()
|
configure_access_logging()
|
||||||
|
|
||||||
|
|
||||||
app = Sanic("cista", strict_slashes=True)
|
app = Sanic("cista", strict_slashes=True)
|
||||||
app.router.ALLOWED_METHODS = (
|
app.router.ALLOWED_METHODS = (
|
||||||
*app.router.ALLOWED_METHODS,
|
*app.router.ALLOWED_METHODS,
|
||||||
@@ -43,8 +56,8 @@ configure_main_logging()
|
|||||||
|
|
||||||
@app.on_request
|
@app.on_request
|
||||||
async def use_session(req):
|
async def use_session(req):
|
||||||
req.ctx._log_start = time.perf_counter()
|
req.ctx.log_start = time.perf_counter()
|
||||||
req.ctx._auth_flow = ["session: start"]
|
req.ctx.auth_flow = ["session: start"]
|
||||||
auth.hydrate_request_auth_context(req, source="app.on_request")
|
auth.hydrate_request_auth_context(req, source="app.on_request")
|
||||||
# CSRF protection
|
# CSRF protection
|
||||||
if req.method == "GET" and req.headers.upgrade != "websocket":
|
if req.method == "GET" and req.headers.upgrade != "websocket":
|
||||||
@@ -61,7 +74,7 @@ async def log_access(req, res):
|
|||||||
"""Log HTTP access in a clean single-line format."""
|
"""Log HTTP access in a clean single-line format."""
|
||||||
if req.headers.get("upgrade", "").lower() == "websocket":
|
if req.headers.get("upgrade", "").lower() == "websocket":
|
||||||
return res
|
return res
|
||||||
start = getattr(req.ctx, "_log_start", None)
|
start = getattr(req.ctx, "log_start", None)
|
||||||
duration_ms = (time.perf_counter() - start) * 1000 if start is not None else 0.0
|
duration_ms = (time.perf_counter() - start) * 1000 if start is not None else 0.0
|
||||||
client = req.client_ip or "-"
|
client = req.client_ip or "-"
|
||||||
host = req.host or "-"
|
host = req.host or "-"
|
||||||
@@ -71,7 +84,7 @@ async def log_access(req, res):
|
|||||||
if isinstance(qs, bytes):
|
if isinstance(qs, bytes):
|
||||||
qs = qs.decode(errors="replace")
|
qs = qs.decode(errors="replace")
|
||||||
path = f"{path}?{qs}"
|
path = f"{path}?{qs}"
|
||||||
extra = getattr(req.ctx, "_log_extra", None)
|
extra = getattr(req.ctx, "log_extra", None)
|
||||||
line = format_access_log(
|
line = format_access_log(
|
||||||
client, res.status, req.method, host, path, duration_ms, extra=extra
|
client, res.status, req.method, host, path, duration_ms, extra=extra
|
||||||
)
|
)
|
||||||
@@ -90,7 +103,7 @@ async def forward_sso_cookies(req, res):
|
|||||||
@app.on_response
|
@app.on_response
|
||||||
async def persist_auth_session(req, res):
|
async def persist_auth_session(req, res):
|
||||||
"""Persist a session cookie after successful Authorization-based auth."""
|
"""Persist a session cookie after successful Authorization-based auth."""
|
||||||
username = getattr(req.ctx, "_create_session_username", None)
|
username = getattr(req.ctx, "create_session_username", None)
|
||||||
if not username or res.status >= 400:
|
if not username or res.status >= 400:
|
||||||
return
|
return
|
||||||
existing = getattr(req.ctx, "session", None)
|
existing = getattr(req.ctx, "session", None)
|
||||||
@@ -126,14 +139,25 @@ async def main_start(app):
|
|||||||
watching.start(app)
|
watching.start(app)
|
||||||
|
|
||||||
|
|
||||||
|
@app.after_server_start
|
||||||
|
async def main_after_start(app):
|
||||||
|
_ = app
|
||||||
|
onlyoffice.log_reachable_info()
|
||||||
|
|
||||||
|
|
||||||
# Sanic sometimes fails to execute after_server_stop, so we do it before instead (potentially interrupting handlers)
|
# Sanic sometimes fails to execute after_server_stop, so we do it before instead (potentially interrupting handlers)
|
||||||
@app.before_server_stop
|
@app.before_server_stop
|
||||||
async def main_stop(app):
|
async def main_stop(app):
|
||||||
watching.stop(app)
|
async with asyncio.TaskGroup() as tg:
|
||||||
await shutdown_preview_workers()
|
tg.create_task(asyncio.to_thread(watching.stop, app))
|
||||||
app.ctx.threadexec.shutdown()
|
tg.create_task(onlyoffice.close_oo_client())
|
||||||
app.ctx.zipexec.shutdown(cancel_futures=True)
|
tg.create_task(shutdown_preview_workers())
|
||||||
await sso.close_client()
|
tg.create_task(sso.close_client())
|
||||||
|
|
||||||
|
async with asyncio.TaskGroup() as tg:
|
||||||
|
tg.create_task(asyncio.to_thread(app.ctx.threadexec.shutdown))
|
||||||
|
tg.create_task(asyncio.to_thread(app.ctx.zipexec.shutdown, cancel_futures=True))
|
||||||
|
|
||||||
logger.debug("Cista worker threads all finished")
|
logger.debug("Cista worker threads all finished")
|
||||||
|
|
||||||
|
|
||||||
@@ -236,6 +260,7 @@ async def wwwroot(req, path=""):
|
|||||||
|
|
||||||
@app.route("/favicon.ico", methods=["GET", "HEAD"])
|
@app.route("/favicon.ico", methods=["GET", "HEAD"])
|
||||||
async def favicon(req):
|
async def favicon(req):
|
||||||
|
_ = req
|
||||||
# Browsers keep asking for it when viewing files (not HTML with icon link)
|
# Browsers keep asking for it when viewing files (not HTML with icon link)
|
||||||
return redirect("/assets/logo-ctv8tVwU.svg", status=308)
|
return redirect("/assets/logo-ctv8tVwU.svg", status=308)
|
||||||
|
|
||||||
|
|||||||
+38
-68
@@ -2,22 +2,21 @@ import base64
|
|||||||
import binascii
|
import binascii
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import re
|
|
||||||
import secrets
|
import secrets
|
||||||
import struct
|
import struct
|
||||||
from pathlib import PurePosixPath
|
from pathlib import PurePosixPath
|
||||||
from time import time
|
from time import time
|
||||||
from unicodedata import normalize
|
|
||||||
|
|
||||||
import argon2
|
|
||||||
import msgspec
|
import msgspec
|
||||||
|
from Crypto.Hash import MD4
|
||||||
from html5tagger import Document
|
from html5tagger import Document
|
||||||
from sanic import Blueprint, html, json, redirect
|
from sanic import Blueprint, html, json, redirect
|
||||||
from sanic.exceptions import BadRequest, Forbidden, Unauthorized
|
from sanic.exceptions import BadRequest, Forbidden, Unauthorized
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
|
|
||||||
from cista import config, session, sharefs
|
from cista import config, session, sharefs
|
||||||
from cista.util import pwgen
|
from cista import sso as _sso_module
|
||||||
|
from cista.util import pwgen, pwhash
|
||||||
from cista.util.filename import sanitize
|
from cista.util.filename import sanitize
|
||||||
|
|
||||||
_LOGIN_PAGE_CSS = """\
|
_LOGIN_PAGE_CSS = """\
|
||||||
@@ -175,16 +174,8 @@ form.onsubmit = async (e) => {
|
|||||||
};
|
};
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Import for SSO validation (lazily loaded to avoid circular imports)
|
|
||||||
_sso_module = None
|
|
||||||
|
|
||||||
|
|
||||||
def _get_sso():
|
def _get_sso():
|
||||||
global _sso_module
|
|
||||||
if _sso_module is None:
|
|
||||||
from cista import sso
|
|
||||||
|
|
||||||
_sso_module = sso
|
|
||||||
return _sso_module
|
return _sso_module
|
||||||
|
|
||||||
|
|
||||||
@@ -202,13 +193,13 @@ def _set_auth_failure_log(request, auth_flow: list[str]) -> None:
|
|||||||
value = request.headers.get(header)
|
value = request.headers.get(header)
|
||||||
if value:
|
if value:
|
||||||
parts.append(f"{label}={value}")
|
parts.append(f"{label}={value}")
|
||||||
request.ctx._log_extra = " | ".join(parts)
|
request.ctx.log_extra = " | ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
def hydrate_request_auth_context(request, *, source: str) -> None:
|
def hydrate_request_auth_context(request, *, source: str) -> None:
|
||||||
auth_flow = getattr(request.ctx, "_auth_flow", None)
|
auth_flow = getattr(request.ctx, "auth_flow", None)
|
||||||
if auth_flow is None:
|
if auth_flow is None:
|
||||||
auth_flow = request.ctx._auth_flow = []
|
auth_flow = request.ctx.auth_flow = []
|
||||||
|
|
||||||
if hasattr(request.ctx, "session"):
|
if hasattr(request.ctx, "session"):
|
||||||
# Already hydrated by an earlier caller (e.g., use_session middleware)
|
# Already hydrated by an earlier caller (e.g., use_session middleware)
|
||||||
@@ -234,9 +225,6 @@ def hydrate_request_auth_context(request, *, source: str) -> None:
|
|||||||
auth_flow.append(f"session:{source}(bad-jwt)")
|
auth_flow.append(f"session:{source}(bad-jwt)")
|
||||||
|
|
||||||
|
|
||||||
_argon = argon2.PasswordHasher()
|
|
||||||
_droppyhash = re.compile(r"^([a-f0-9]{64})\$([a-f0-9]{8})$")
|
|
||||||
|
|
||||||
_AUTH_REALM = "cista"
|
_AUTH_REALM = "cista"
|
||||||
_AUTH_CACHE_TTL = 10
|
_AUTH_CACHE_TTL = 10
|
||||||
_auth_cache: dict[str, tuple[float, config.User]] = {}
|
_auth_cache: dict[str, tuple[float, config.User]] = {}
|
||||||
@@ -280,6 +268,7 @@ def _log_webdav_user_agent_once(request, user_agent: str):
|
|||||||
|
|
||||||
|
|
||||||
def _build_ua_auth_headers(request, *, include_hint=False) -> dict[str, str]:
|
def _build_ua_auth_headers(request, *, include_hint=False) -> dict[str, str]:
|
||||||
|
_ = include_hint
|
||||||
user_agent = request.headers.get("user-agent", "")
|
user_agent = request.headers.get("user-agent", "")
|
||||||
_log_webdav_user_agent_once(request, user_agent)
|
_log_webdav_user_agent_once(request, user_agent)
|
||||||
if _is_windows_auth_client(user_agent):
|
if _is_windows_auth_client(user_agent):
|
||||||
@@ -448,12 +437,6 @@ def _ntlmv2_verify(
|
|||||||
nt_response: bytes,
|
nt_response: bytes,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Verify an NTLMv2 response using the plaintext token secret as the password."""
|
"""Verify an NTLMv2 response using the plaintext token secret as the password."""
|
||||||
try:
|
|
||||||
from Crypto.Hash import MD4
|
|
||||||
except ImportError:
|
|
||||||
logger.error("pycryptodome MD4 not available, cannot verify NTLM")
|
|
||||||
return False
|
|
||||||
|
|
||||||
if len(nt_response) < 16:
|
if len(nt_response) < 16:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -517,47 +500,30 @@ def _ntlmv2_verify(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _pwnorm(password):
|
|
||||||
return normalize("NFC", password).strip().encode()
|
|
||||||
|
|
||||||
|
|
||||||
def _cache_key(username: str, password: str) -> str:
|
def _cache_key(username: str, password: str) -> str:
|
||||||
return hashlib.sha256(f"{username}\x00{password}".encode()).hexdigest()
|
return hashlib.sha256(f"{username}\x00{password}".encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def login(username: str, password: str):
|
def login(username: str, password: str):
|
||||||
|
normalized_username = pwhash.normalize_secret(username).decode()
|
||||||
cache_key = _cache_key(username, password)
|
cache_key = _cache_key(username, password)
|
||||||
cached = _auth_cache.get(cache_key)
|
cached = _auth_cache.get(cache_key)
|
||||||
if cached:
|
if cached:
|
||||||
ts, user = cached
|
ts, user = cached
|
||||||
if time() - ts < _AUTH_CACHE_TTL:
|
if time() - ts < _AUTH_CACHE_TTL:
|
||||||
return user
|
current = config.config.users.get(normalized_username)
|
||||||
|
if current and current.hash == user.hash:
|
||||||
|
return current
|
||||||
del _auth_cache[cache_key]
|
del _auth_cache[cache_key]
|
||||||
|
|
||||||
un = _pwnorm(username)
|
|
||||||
pw = _pwnorm(password)
|
|
||||||
try:
|
try:
|
||||||
u = config.config.users[un.decode()]
|
u = config.config.users[normalized_username]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise ValueError("Invalid username") from None
|
raise ValueError("Invalid username") from None
|
||||||
# Verify password
|
# Verify password
|
||||||
need_rehash = False
|
need_rehash = pwhash.verify_hash(
|
||||||
if not u.hash:
|
u.hash, username=normalized_username, password=password
|
||||||
raise ValueError("Account disabled")
|
)
|
||||||
if (m := _droppyhash.match(u.hash)) is not None:
|
|
||||||
h, s = m.groups()
|
|
||||||
h2 = hmac.digest(pw + s.encode() + un, b"", "sha256").hex()
|
|
||||||
if not hmac.compare_digest(h, h2):
|
|
||||||
raise ValueError("Invalid password")
|
|
||||||
# Droppy hashes are weak, do a hash update
|
|
||||||
need_rehash = True
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
_argon.verify(u.hash, pw)
|
|
||||||
except Exception:
|
|
||||||
raise ValueError("Invalid password") from None
|
|
||||||
if _argon.check_needs_rehash(u.hash):
|
|
||||||
need_rehash = True
|
|
||||||
# Login successful
|
# Login successful
|
||||||
if need_rehash:
|
if need_rehash:
|
||||||
set_password(u, password)
|
set_password(u, password)
|
||||||
@@ -568,7 +534,7 @@ def login(username: str, password: str):
|
|||||||
|
|
||||||
|
|
||||||
def set_password(user: config.User, password: str):
|
def set_password(user: config.User, password: str):
|
||||||
user.hash = _argon.hash(_pwnorm(password))
|
pwhash.set_password(user, password)
|
||||||
_auth_cache.clear()
|
_auth_cache.clear()
|
||||||
|
|
||||||
|
|
||||||
@@ -670,11 +636,12 @@ async def _token_auth_login(request, *, privileged=False):
|
|||||||
ctx = data.get("ctx", {}) if isinstance(data, dict) else {}
|
ctx = data.get("ctx", {}) if isinstance(data, dict) else {}
|
||||||
user_info = ctx.get("user", {}) if isinstance(ctx, dict) else {}
|
user_info = ctx.get("user", {}) if isinstance(ctx, dict) else {}
|
||||||
request.ctx.username = user_info.get("display_name", "")
|
request.ctx.username = user_info.get("display_name", "")
|
||||||
return True
|
|
||||||
except Forbidden:
|
except Forbidden:
|
||||||
raise
|
raise
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
if token.username:
|
if token.username:
|
||||||
user = config.config.users.get(token.username)
|
user = config.config.users.get(token.username)
|
||||||
@@ -840,12 +807,13 @@ async def _ntlm_auth_login(request, *, privileged=False):
|
|||||||
token.sso_user_id,
|
token.sso_user_id,
|
||||||
tid[:8],
|
tid[:8],
|
||||||
)
|
)
|
||||||
return True
|
|
||||||
except Forbidden:
|
except Forbidden:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("NTLM SSO check failed: %s", e)
|
logger.warning("NTLM SSO check failed: %s", e)
|
||||||
continue
|
continue
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
if token.username:
|
if token.username:
|
||||||
user = config.config.users.get(token.username)
|
user = config.config.users.get(token.username)
|
||||||
@@ -865,7 +833,7 @@ async def _ntlm_auth_login(request, *, privileged=False):
|
|||||||
request.ctx.user = user
|
request.ctx.user = user
|
||||||
request.ctx.auth_token_id = tid
|
request.ctx.auth_token_id = tid
|
||||||
request.ctx.auth_token = token
|
request.ctx.auth_token = token
|
||||||
request.ctx._create_session_username = token.username
|
request.ctx.create_session_username = token.username
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"NTLM auth success for local user %s (token=%s...)",
|
"NTLM auth success for local user %s (token=%s...)",
|
||||||
token.username,
|
token.username,
|
||||||
@@ -914,7 +882,7 @@ async def verify(request, *, privileged=False):
|
|||||||
scheme = auth_header.split()[0].lower() if has_auth_header else None
|
scheme = auth_header.split()[0].lower() if has_auth_header else None
|
||||||
|
|
||||||
# Concise auth flow for diagnostics (populated by use_session + verify)
|
# Concise auth flow for diagnostics (populated by use_session + verify)
|
||||||
auth_flow = list(getattr(request.ctx, "_auth_flow", ["session:skipped"]))
|
auth_flow = list(getattr(request.ctx, "auth_flow", ["session:skipped"]))
|
||||||
tried: list[str] = []
|
tried: list[str] = []
|
||||||
|
|
||||||
sso = _get_sso()
|
sso = _get_sso()
|
||||||
@@ -927,7 +895,6 @@ async def verify(request, *, privileged=False):
|
|||||||
try:
|
try:
|
||||||
perm = "cista:admin" if privileged else "cista:login"
|
perm = "cista:admin" if privileged else "cista:login"
|
||||||
await sso.validate_sso_request(request, perm=perm)
|
await sso.validate_sso_request(request, perm=perm)
|
||||||
return
|
|
||||||
except Unauthorized as e:
|
except Unauthorized as e:
|
||||||
auth_flow.append(f"tried={','.join(tried)} result=failed")
|
auth_flow.append(f"tried={','.join(tried)} result=failed")
|
||||||
_set_auth_failure_log(request, auth_flow)
|
_set_auth_failure_log(request, auth_flow)
|
||||||
@@ -936,6 +903,8 @@ async def verify(request, *, privileged=False):
|
|||||||
headers=_build_ua_auth_headers(request),
|
headers=_build_ua_auth_headers(request),
|
||||||
quiet=True,
|
quiet=True,
|
||||||
) from e
|
) from e
|
||||||
|
else:
|
||||||
|
return
|
||||||
tried.append("sso")
|
tried.append("sso")
|
||||||
perm = "cista:admin" if privileged else "cista:login"
|
perm = "cista:admin" if privileged else "cista:login"
|
||||||
await sso.validate_sso_request(request, perm=perm)
|
await sso.validate_sso_request(request, perm=perm)
|
||||||
@@ -986,10 +955,10 @@ async def verify(request, *, privileged=False):
|
|||||||
user = None
|
user = None
|
||||||
else:
|
else:
|
||||||
if user is not None:
|
if user is not None:
|
||||||
if getattr(request.ctx, "_create_session_username", None) is None:
|
if getattr(request.ctx, "create_session_username", None) is None:
|
||||||
username = getattr(request.ctx, "username", None)
|
username = getattr(request.ctx, "username", None)
|
||||||
if username:
|
if username:
|
||||||
request.ctx._create_session_username = username
|
request.ctx.create_session_username = username
|
||||||
return
|
return
|
||||||
# Auth header present but invalid → try session fallback
|
# Auth header present but invalid → try session fallback
|
||||||
tried.append("session")
|
tried.append("session")
|
||||||
@@ -1113,6 +1082,7 @@ async def login_page(request):
|
|||||||
|
|
||||||
def _login_success_page(username: str) -> str:
|
def _login_success_page(username: str) -> str:
|
||||||
"""Minimal page that signals auth-success to parent iframe."""
|
"""Minimal page that signals auth-success to parent iframe."""
|
||||||
|
_ = username
|
||||||
return str(
|
return str(
|
||||||
Document().script_("window.parent.postMessage({type:'auth-success'},'*')")
|
Document().script_("window.parent.postMessage({type:'auth-success'},'*')")
|
||||||
)
|
)
|
||||||
@@ -1127,13 +1097,16 @@ async def login_post(request):
|
|||||||
else:
|
else:
|
||||||
username = request.form["username"][0]
|
username = request.form["username"][0]
|
||||||
password = request.form["password"][0]
|
password = request.form["password"][0]
|
||||||
if not username or not password:
|
|
||||||
raise KeyError
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise BadRequest(
|
raise BadRequest(
|
||||||
"Missing username or password",
|
"Missing username or password",
|
||||||
context={"redirect": "/login"},
|
context={"redirect": "/login"},
|
||||||
) from None
|
) from None
|
||||||
|
if not username or not password:
|
||||||
|
raise BadRequest(
|
||||||
|
"Missing username or password",
|
||||||
|
context={"redirect": "/login"},
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
user = login(username, password)
|
user = login(username, password)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
@@ -1172,12 +1145,12 @@ async def change_password(request):
|
|||||||
username = request.form["username"][0]
|
username = request.form["username"][0]
|
||||||
pwchange = request.form["passwordChange"][0]
|
pwchange = request.form["passwordChange"][0]
|
||||||
password = request.form["password"][0]
|
password = request.form["password"][0]
|
||||||
if not username or not password:
|
|
||||||
raise KeyError
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise BadRequest(
|
raise BadRequest(
|
||||||
"Missing username, passwordChange or password",
|
"Missing username, passwordChange or password",
|
||||||
) from None
|
) from None
|
||||||
|
if not username or not password:
|
||||||
|
raise BadRequest("Missing username, passwordChange or password")
|
||||||
try:
|
try:
|
||||||
user = login(username, password)
|
user = login(username, password)
|
||||||
set_password(user, pwchange)
|
set_password(user, pwchange)
|
||||||
@@ -1220,16 +1193,15 @@ async def create_user(request):
|
|||||||
username = request.form["username"][0]
|
username = request.form["username"][0]
|
||||||
password = request.form.get("password", [None])[0]
|
password = request.form.get("password", [None])[0]
|
||||||
privileged = request.form.get("privileged", ["false"])[0].lower() == "true"
|
privileged = request.form.get("privileged", ["false"])[0].lower() == "true"
|
||||||
|
except KeyError as e:
|
||||||
|
raise BadRequest("Missing fields") from e
|
||||||
if not username or not username.isidentifier():
|
if not username or not username.isidentifier():
|
||||||
raise ValueError("Invalid username")
|
raise BadRequest("Invalid username")
|
||||||
except (KeyError, ValueError) as e:
|
|
||||||
raise BadRequest(str(e)) from e
|
|
||||||
if username in config.config.users:
|
if username in config.config.users:
|
||||||
raise BadRequest("User already exists")
|
raise BadRequest("User already exists")
|
||||||
if not password:
|
if not password:
|
||||||
password = pwgen.generate()
|
password = pwgen.generate()
|
||||||
changes = {"privileged": privileged}
|
changes = {"privileged": privileged, "password": password}
|
||||||
changes["hash"] = _argon.hash(_pwnorm(password))
|
|
||||||
try:
|
try:
|
||||||
config.update_user(username, changes)
|
config.update_user(username, changes)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -1256,8 +1228,6 @@ async def update_user(request, username):
|
|||||||
if changes["password"] == "":
|
if changes["password"] == "":
|
||||||
changes["password"] = pwgen.generate()
|
changes["password"] = pwgen.generate()
|
||||||
password_response = changes["password"]
|
password_response = changes["password"]
|
||||||
changes["hash"] = _argon.hash(_pwnorm(changes["password"]))
|
|
||||||
del changes["password"]
|
|
||||||
if not changes:
|
if not changes:
|
||||||
return json({"message": "No changes"})
|
return json({"message": "No changes"})
|
||||||
try:
|
try:
|
||||||
|
|||||||
+3
-3
@@ -14,6 +14,8 @@ from typing import Concatenate, Literal, ParamSpec
|
|||||||
import msgspec
|
import msgspec
|
||||||
import msgspec.toml
|
import msgspec.toml
|
||||||
|
|
||||||
|
from .util import pwhash
|
||||||
|
|
||||||
|
|
||||||
class Config(msgspec.Struct):
|
class Config(msgspec.Struct):
|
||||||
path: Path
|
path: Path
|
||||||
@@ -199,9 +201,7 @@ def update_user(conf: Config, name: str, changes: dict) -> Config:
|
|||||||
except KeyError:
|
except KeyError:
|
||||||
u = User()
|
u = User()
|
||||||
if "password" in changes:
|
if "password" in changes:
|
||||||
from . import auth
|
pwhash.set_password(u, changes["password"])
|
||||||
|
|
||||||
auth.set_password(u, changes["password"])
|
|
||||||
del changes["password"]
|
del changes["password"]
|
||||||
udict = msgspec.to_builtins(u, enc_hook=enc_hook)
|
udict = msgspec.to_builtins(u, enc_hook=enc_hook)
|
||||||
udict.update(changes)
|
udict.update(changes)
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Patched OnlyOffice Document Server with configurable converter worker count.
|
||||||
|
#
|
||||||
|
# The Community Edition hardcodes the document converter to 1 worker,
|
||||||
|
# which creates a severe bottleneck under concurrent load.
|
||||||
|
# This image patches the open-source license.js to spawn a configurable
|
||||||
|
# number of converter workers (default 8).
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build -t onlyoffice-cista docker/onlyoffice-converter-patch
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker run -d -p 8988:80 \
|
||||||
|
# -e WORKERS=16 \
|
||||||
|
# -e JWT_SECRET=your-strong-secret \
|
||||||
|
# --name onlyoffice onlyoffice-cista
|
||||||
|
#
|
||||||
|
# JWT:
|
||||||
|
# Set JWT_SECRET to the same value you pass to Cista as ONLYOFFICE_JWT_SECRET.
|
||||||
|
# OnlyOffice will enable token validation automatically.
|
||||||
|
#
|
||||||
|
# The ONLYOFFICE_VERSION build arg lets you target a specific release.
|
||||||
|
|
||||||
|
ARG ONLYOFFICE_VERSION=9.3.1
|
||||||
|
|
||||||
|
FROM onlyoffice/documentserver:${ONLYOFFICE_VERSION}
|
||||||
|
|
||||||
|
# Prevent interactive apt prompts
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# Install Node.js, npm, and git so we can run the FileConverter from source.
|
||||||
|
RUN apt-get update -qq && \
|
||||||
|
apt-get install -y -qq --no-install-recommends \
|
||||||
|
nodejs \
|
||||||
|
npm \
|
||||||
|
git \
|
||||||
|
ca-certificates && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Clone the open-source server components (shallow, ~15 MB).
|
||||||
|
# The master branch is used because the Linux/web tags are not published
|
||||||
|
# in the server repo; the license.js file has been stable for years.
|
||||||
|
RUN git clone --depth 1 https://github.com/ONLYOFFICE/server.git /opt/oo-server
|
||||||
|
|
||||||
|
# Patch license.js so the converter worker count is read from an env var
|
||||||
|
# instead of being hardcoded to 1.
|
||||||
|
RUN sed -i \
|
||||||
|
's/count: 1,/count: parseInt(process.env.WORKERS, 10) || 8,/' \
|
||||||
|
/opt/oo-server/Common/sources/license.js
|
||||||
|
|
||||||
|
# Install npm dependencies for the modules the FileConverter touches.
|
||||||
|
# DocService deps are also needed because converter.js pulls in baseConnector.
|
||||||
|
RUN cd /opt/oo-server/Common && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/FileConverter && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/DocService && npm ci --no-audit --no-fund
|
||||||
|
|
||||||
|
# Back up the compiled pkg binary and replace it with our wrapper.
|
||||||
|
RUN mv /var/www/onlyoffice/documentserver/server/FileConverter/converter \
|
||||||
|
/var/www/onlyoffice/documentserver/server/FileConverter/converter.orig
|
||||||
|
|
||||||
|
COPY converter-wrapper.sh /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
RUN chmod +x /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
|
||||||
|
# Default worker count (override at runtime with -e WORKERS=16).
|
||||||
|
ENV WORKERS=8
|
||||||
|
|
||||||
|
# Use our custom entrypoint to persist the env var to a file that the
|
||||||
|
# non-root converter process (user=ds) can read.
|
||||||
|
COPY entrypoint.sh /app/ds/run-document-server-patched.sh
|
||||||
|
RUN chmod +x /app/ds/run-document-server-patched.sh
|
||||||
|
ENTRYPOINT ["/app/ds/run-document-server-patched.sh"]
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Wrapper that runs the OnlyOffice FileConverter from patched Node.js source.
|
||||||
|
# Replaces the compiled pkg binary shipped with the Community Edition.
|
||||||
|
|
||||||
|
# The env var is not passed through supervisor to the 'ds' user, so we read
|
||||||
|
# it from a file written by the custom entrypoint.
|
||||||
|
if [ -z "${WORKERS}" ] && [ -r /tmp/oo-converter-workers.txt ]; then
|
||||||
|
export WORKERS=$(cat /tmp/oo-converter-workers.txt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd /opt/oo-server/FileConverter || exit 1
|
||||||
|
|
||||||
|
export NODE_ENV=production-linux
|
||||||
|
export NODE_CONFIG_DIR=/etc/onlyoffice/documentserver
|
||||||
|
export NODE_DISABLE_COLORS=1
|
||||||
|
export APPLICATION_NAME=onlyoffice
|
||||||
|
export LD_LIBRARY_PATH=/var/www/onlyoffice/documentserver/server/FileConverter/bin
|
||||||
|
|
||||||
|
exec node sources/convertermaster.js "$@"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Custom entrypoint that persists WORKERS to a file readable by
|
||||||
|
# the non-root user that supervisor uses to run the converter.
|
||||||
|
|
||||||
|
echo "${WORKERS:-8}" > /tmp/oo-converter-workers.txt
|
||||||
|
chmod 644 /tmp/oo-converter-workers.txt
|
||||||
|
|
||||||
|
exec /app/ds/run-document-server.sh "$@"
|
||||||
+6
-24
@@ -76,7 +76,7 @@ async def upload_file_chunk(request, name):
|
|||||||
size_after = upload_info.get("size_after")
|
size_after = upload_info.get("size_after")
|
||||||
if size_before is not None and size_after is not None and size_before != size_after:
|
if size_before is not None and size_after is not None and size_before != size_after:
|
||||||
extras.append("resized")
|
extras.append("resized")
|
||||||
request.ctx._log_extra = " ".join(extras) if extras else None
|
request.ctx.log_extra = " ".join(extras) if extras else None
|
||||||
real_rel = PurePosixPath(path.relative_to(config.config.path.resolve()).as_posix())
|
real_rel = PurePosixPath(path.relative_to(config.config.path.resolve()).as_posix())
|
||||||
watching.notify_change(real_rel, *real_rel.parents)
|
watching.notify_change(real_rel, *real_rel.parents)
|
||||||
return json(
|
return json(
|
||||||
@@ -197,38 +197,18 @@ async def copy_or_move(request, name=""):
|
|||||||
|
|
||||||
def _apply():
|
def _apply():
|
||||||
for op_name, op_keys in (("cp", cp_keys), ("mv", mv_keys)):
|
for op_name, op_keys in (("cp", cp_keys), ("mv", mv_keys)):
|
||||||
op_multi = len(op_keys) > 1
|
|
||||||
for key in op_keys:
|
for key in op_keys:
|
||||||
try:
|
try:
|
||||||
src_rel = key_paths[key]
|
src_rel = key_paths[key]
|
||||||
src_abs = _resolve_from_relpath(src_rel, request=request)
|
src_abs = _resolve_from_relpath(src_rel, request=request)
|
||||||
|
|
||||||
if op_multi:
|
if dst_is_dir:
|
||||||
if not dst_is_dir:
|
|
||||||
raise BadRequest(
|
|
||||||
"Destination must be an existing directory for multiple keys"
|
|
||||||
)
|
|
||||||
dst_item_rel = (
|
|
||||||
dst_rel / src_rel.name
|
|
||||||
if dst_rel.parts
|
|
||||||
else PurePosixPath(src_rel.name)
|
|
||||||
)
|
|
||||||
elif dst_is_dir:
|
|
||||||
dst_item_rel = (
|
dst_item_rel = (
|
||||||
dst_rel / src_rel.name
|
dst_rel / src_rel.name
|
||||||
if dst_rel.parts
|
if dst_rel.parts
|
||||||
else PurePosixPath(src_rel.name)
|
else PurePosixPath(src_rel.name)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
if not dst_rel.parts:
|
|
||||||
raise BadRequest("Destination file path is required")
|
|
||||||
parent_abs = dst_abs.parent
|
|
||||||
if not parent_abs.is_dir():
|
|
||||||
raise BadRequest("Destination parent folder does not exist")
|
|
||||||
if src_abs.is_dir() and dst_exists and dst_abs.is_file():
|
|
||||||
raise BadRequest(
|
|
||||||
"Cannot move/copy a directory to an existing file"
|
|
||||||
)
|
|
||||||
dst_item_rel = dst_rel
|
dst_item_rel = dst_rel
|
||||||
|
|
||||||
dst_item_abs = _resolve_from_relpath(dst_item_rel, request=request)
|
dst_item_abs = _resolve_from_relpath(dst_item_rel, request=request)
|
||||||
@@ -301,6 +281,8 @@ async def head_file(request, name=""):
|
|||||||
@bp.route("/", methods=["OPTIONS"], name="options_root", strict_slashes=False)
|
@bp.route("/", methods=["OPTIONS"], name="options_root", strict_slashes=False)
|
||||||
@bp.route("/<name:path>", methods=["OPTIONS"], name="options_path")
|
@bp.route("/<name:path>", methods=["OPTIONS"], name="options_path")
|
||||||
async def dav_options(request, name=""):
|
async def dav_options(request, name=""):
|
||||||
|
_ = request
|
||||||
|
_ = name
|
||||||
return HTTPResponse(
|
return HTTPResponse(
|
||||||
status=200,
|
status=200,
|
||||||
headers={
|
headers={
|
||||||
@@ -362,7 +344,7 @@ async def dav_copy(request, name=""):
|
|||||||
dst_rel, dst_abs = _parse_webdav_destination(dest_header, request=request)
|
dst_rel, dst_abs = _parse_webdav_destination(dest_header, request=request)
|
||||||
if auth.request_share_token(request) is not None and not dst_rel.parts:
|
if auth.request_share_token(request) is not None and not dst_rel.parts:
|
||||||
raise BadRequest("Destination cannot be virtual root")
|
raise BadRequest("Destination cannot be virtual root")
|
||||||
request.ctx._log_extra = f"→ {dst_rel}"
|
request.ctx.log_extra = f"→ {dst_rel}"
|
||||||
if not src_abs.exists():
|
if not src_abs.exists():
|
||||||
raise NotFound(f"Source not found: {name}")
|
raise NotFound(f"Source not found: {name}")
|
||||||
if src_abs == dst_abs:
|
if src_abs == dst_abs:
|
||||||
@@ -401,7 +383,7 @@ async def dav_move(request, name=""):
|
|||||||
dst_rel, dst_abs = _parse_webdav_destination(dest_header, request=request)
|
dst_rel, dst_abs = _parse_webdav_destination(dest_header, request=request)
|
||||||
if auth.request_share_token(request) is not None and not dst_rel.parts:
|
if auth.request_share_token(request) is not None and not dst_rel.parts:
|
||||||
raise BadRequest("Destination cannot be virtual root")
|
raise BadRequest("Destination cannot be virtual root")
|
||||||
request.ctx._log_extra = f"→ {dst_rel}"
|
request.ctx.log_extra = f"→ {dst_rel}"
|
||||||
if not src_abs.exists():
|
if not src_abs.exists():
|
||||||
raise NotFound(f"Source not found: {name}")
|
raise NotFound(f"Source not found: {name}")
|
||||||
if src_abs == dst_abs:
|
if src_abs == dst_abs:
|
||||||
|
|||||||
+161
-24
@@ -10,12 +10,14 @@ Environment requirements:
|
|||||||
reachable from the container (usually the docker bridge IP).
|
reachable from the container (usually the docker bridge IP).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
import socketserver
|
import socketserver
|
||||||
import subprocess
|
import subprocess
|
||||||
import threading
|
import threading
|
||||||
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from http.server import SimpleHTTPRequestHandler
|
from http.server import SimpleHTTPRequestHandler
|
||||||
@@ -23,20 +25,29 @@ from pathlib import Path
|
|||||||
from time import perf_counter
|
from time import perf_counter
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
import httpx
|
||||||
import jwt
|
import jwt
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
|
|
||||||
|
from cista import config
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Configuration helpers
|
# Configuration helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
_httpx_client: httpx.AsyncClient | None = None
|
||||||
|
|
||||||
|
|
||||||
def _get_onlyoffice_url() -> str:
|
def _get_onlyoffice_url() -> str:
|
||||||
return os.environ.get("ONLYOFFICE_URL", "http://localhost:8080")
|
return os.environ.get("ONLYOFFICE_CISTA_URL", "http://localhost:8988")
|
||||||
|
|
||||||
|
|
||||||
def _get_jwt_secret() -> str | None:
|
def _get_jwt_secret() -> str:
|
||||||
return os.environ.get("ONLYOFFICE_JWT_SECRET") or None
|
return (
|
||||||
|
os.environ.get("ONLYOFFICE_JWT_SECRET")
|
||||||
|
or config.derived_secret("onlyoffice", size=16).hex()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_callback_host() -> str:
|
def _get_callback_host() -> str:
|
||||||
@@ -62,19 +73,142 @@ def _get_callback_host() -> str:
|
|||||||
return "127.0.0.1"
|
return "127.0.0.1"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Async HTTP client
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def get_httpx_client() -> httpx.AsyncClient:
|
||||||
|
"""Return the shared async HTTP client for OnlyOffice requests."""
|
||||||
|
global _httpx_client
|
||||||
|
if _httpx_client is None:
|
||||||
|
_httpx_client = httpx.AsyncClient()
|
||||||
|
return _httpx_client
|
||||||
|
|
||||||
|
|
||||||
|
async def close_oo_client() -> None:
|
||||||
|
"""Close the shared async HTTP client."""
|
||||||
|
global _httpx_client
|
||||||
|
if _httpx_client is not None:
|
||||||
|
await _httpx_client.aclose()
|
||||||
|
_httpx_client = None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Availability check
|
# Availability check
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def is_available() -> bool:
|
def _probe_status() -> tuple[bool, bool, str | None]:
|
||||||
"""Return True if the configured OnlyOffice Document Server is reachable."""
|
"""Return (ok, responded, detail) for a lightweight reachability probe."""
|
||||||
url = _get_onlyoffice_url()
|
url = _get_onlyoffice_url().rstrip("/") + "/ConvertService.ashx"
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=3) as resp: # noqa: S310
|
with urllib.request.urlopen(url, timeout=2) as resp: # noqa: S310
|
||||||
return resp.status == 200
|
status = resp.status
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
status = e.code
|
||||||
|
except Exception:
|
||||||
|
return False, False, None
|
||||||
|
|
||||||
|
if status in (200, 405):
|
||||||
|
return True, True, None
|
||||||
|
if status >= 500:
|
||||||
|
return False, True, f"HTTP {status}"
|
||||||
|
return False, True, f"HTTP {status}"
|
||||||
|
|
||||||
|
|
||||||
|
def log_reachable_info() -> None:
|
||||||
|
"""Log info on success, warning on responded probe errors, silent on no-response."""
|
||||||
|
ok, responded, detail = _probe_status()
|
||||||
|
if ok:
|
||||||
|
logger.info("Using OnlyOffice document server at %s", _get_onlyoffice_url())
|
||||||
|
elif responded:
|
||||||
|
suffix = f": {detail}" if detail else ""
|
||||||
|
logger.warning("OnlyOffice probe failed%s", suffix)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_docker(confdir: Path | None = None) -> int:
|
||||||
|
"""Build and run the patched OnlyOffice Docker image."""
|
||||||
|
if confdir is not None:
|
||||||
|
os.environ["CISTA_HOME"] = confdir.as_posix()
|
||||||
|
config.init_confdir()
|
||||||
|
if config.conffile.exists():
|
||||||
|
config.load_config()
|
||||||
|
else:
|
||||||
|
config.update_config(
|
||||||
|
{
|
||||||
|
"listen": ":8989",
|
||||||
|
"path": Path.home() / "Downloads",
|
||||||
|
"public": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
secret = config.derived_secret("onlyoffice", size=16).hex()
|
||||||
|
docker_dir = Path(__file__).parent / "docker"
|
||||||
|
if not docker_dir.is_dir():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Docker files not found at {docker_dir}. Is the package installed correctly?"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Building OnlyOffice image")
|
||||||
|
build_cmd = ["docker", "build", "-t", "onlyoffice-cista", str(docker_dir)]
|
||||||
|
logger.info("%s", " ".join(build_cmd))
|
||||||
|
result = subprocess.run(build_cmd, check=False, shell=False) # noqa: S603
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("Failed to build OnlyOffice image")
|
||||||
|
|
||||||
|
logger.info("Starting OnlyOffice container")
|
||||||
|
run_cmd = [
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"-p",
|
||||||
|
"8988:80",
|
||||||
|
"-e",
|
||||||
|
f"JWT_SECRET={secret}",
|
||||||
|
"-e",
|
||||||
|
"WORKERS=8",
|
||||||
|
"--name",
|
||||||
|
"onlyoffice-cista",
|
||||||
|
"--restart",
|
||||||
|
"unless-stopped",
|
||||||
|
"onlyoffice-cista",
|
||||||
|
]
|
||||||
|
logger.info("%s", " ".join(run_cmd))
|
||||||
|
result = subprocess.run(run_cmd, check=False, shell=False) # noqa: S603
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("Failed to start OnlyOffice container")
|
||||||
|
logger.info("OnlyOffice is running on http://localhost:8988")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
async def is_available_async(request_timeout: float = 2.0) -> bool:
|
||||||
|
"""Return True if the configured OnlyOffice Document Server is reachable."""
|
||||||
|
url = _get_onlyoffice_url().rstrip("/") + "/ConvertService.ashx"
|
||||||
|
client = get_httpx_client()
|
||||||
|
try:
|
||||||
|
response = await client.get(url, timeout=request_timeout)
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
else:
|
||||||
|
return response.status_code in (200, 405)
|
||||||
|
|
||||||
|
|
||||||
|
_oo_available_cache: tuple[bool, float] | None = None
|
||||||
|
OO_AVAILABILITY_CACHE_TTL = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
async def is_available_cached() -> bool:
|
||||||
|
"""Return cached OnlyOffice availability, refreshed every 30 seconds."""
|
||||||
|
global _oo_available_cache
|
||||||
|
now = perf_counter()
|
||||||
|
if _oo_available_cache is not None:
|
||||||
|
result, timestamp = _oo_available_cache
|
||||||
|
if now - timestamp < OO_AVAILABILITY_CACHE_TTL:
|
||||||
|
return result
|
||||||
|
result = await is_available_async()
|
||||||
|
_oo_available_cache = (result, now)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -121,22 +255,23 @@ def _build_jwt_token(payload: dict) -> str | None:
|
|||||||
return jwt.encode(payload, secret, algorithm="HS256")
|
return jwt.encode(payload, secret, algorithm="HS256")
|
||||||
|
|
||||||
|
|
||||||
def convert_to_png(file_path: Path, timeout: float = 30.0) -> bytes:
|
async def convert_to_png_async(file_path: Path, request_timeout: float = 5.0) -> bytes:
|
||||||
"""Convert *file_path* to PNG using OnlyOffice Document Server.
|
"""Convert *file_path* to PNG using OnlyOffice Document Server (async).
|
||||||
|
|
||||||
Returns the PNG bytes. Raises RuntimeError on failure.
|
Returns the PNG bytes. Raises RuntimeError on failure.
|
||||||
"""
|
"""
|
||||||
oo_url = _get_onlyoffice_url().rstrip("/")
|
oo_url = _get_onlyoffice_url().rstrip("/")
|
||||||
convert_url = f"{oo_url}/ConvertService.ashx"
|
convert_url = f"{oo_url}/ConvertService.ashx"
|
||||||
|
client = get_httpx_client()
|
||||||
|
|
||||||
# Start temporary HTTP server so OnlyOffice can fetch the file
|
# Start temporary HTTP server so OnlyOffice can fetch the file
|
||||||
doc_url, httpd = _serve_file_temporarily(file_path)
|
doc_url, httpd = await asyncio.to_thread(_serve_file_temporarily, file_path)
|
||||||
try:
|
try:
|
||||||
suffix = file_path.suffix.lstrip(".").lower()
|
suffix = file_path.suffix.lstrip(".").lower()
|
||||||
payload = {
|
payload = {
|
||||||
"async": False,
|
"async": False,
|
||||||
"filetype": suffix,
|
"filetype": suffix,
|
||||||
"key": f"cista_{file_path.stat().st_mtime_ns}",
|
"key": f"cista_{(await asyncio.to_thread(file_path.stat)).st_mtime_ns}",
|
||||||
"outputtype": "png",
|
"outputtype": "png",
|
||||||
"title": file_path.name,
|
"title": file_path.name,
|
||||||
"url": doc_url,
|
"url": doc_url,
|
||||||
@@ -145,18 +280,19 @@ def convert_to_png(file_path: Path, timeout: float = 30.0) -> bytes:
|
|||||||
headers = {"Content-Type": "application/json"}
|
headers = {"Content-Type": "application/json"}
|
||||||
token = _build_jwt_token(payload)
|
token = _build_jwt_token(payload)
|
||||||
if token:
|
if token:
|
||||||
|
# Conversion API expects JWT in request body when token checks are enabled.
|
||||||
|
payload["token"] = token
|
||||||
headers["Authorization"] = token
|
headers["Authorization"] = token
|
||||||
|
|
||||||
req = urllib.request.Request( # noqa: S310
|
|
||||||
convert_url,
|
|
||||||
data=json.dumps(payload).encode(),
|
|
||||||
headers=headers,
|
|
||||||
method="POST",
|
|
||||||
)
|
|
||||||
|
|
||||||
t_start = perf_counter()
|
t_start = perf_counter()
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
|
response = await client.post(
|
||||||
body = resp.read()
|
convert_url,
|
||||||
|
content=json.dumps(payload).encode(),
|
||||||
|
headers=headers,
|
||||||
|
timeout=request_timeout,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
body = response.content
|
||||||
t_end = perf_counter()
|
t_end = perf_counter()
|
||||||
|
|
||||||
# Parse XML response
|
# Parse XML response
|
||||||
@@ -176,7 +312,8 @@ def convert_to_png(file_path: Path, timeout: float = 30.0) -> bytes:
|
|||||||
logger.debug("OnlyOffice converted in %.2fs: %s", t_end - t_start, file_url)
|
logger.debug("OnlyOffice converted in %.2fs: %s", t_end - t_start, file_url)
|
||||||
|
|
||||||
# Download converted PNG
|
# Download converted PNG
|
||||||
with urllib.request.urlopen(file_url, timeout=timeout) as png_resp: # noqa: S310
|
png_response = await client.get(file_url, timeout=request_timeout)
|
||||||
return png_resp.read()
|
png_response.raise_for_status()
|
||||||
|
return png_response.content
|
||||||
finally:
|
finally:
|
||||||
httpd.shutdown()
|
await asyncio.to_thread(httpd.shutdown)
|
||||||
|
|||||||
+343
-338
@@ -1,7 +1,5 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import contextlib
|
import contextlib
|
||||||
import gc
|
|
||||||
import io
|
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import struct
|
import struct
|
||||||
import sys
|
import sys
|
||||||
@@ -10,41 +8,28 @@ import urllib.parse
|
|||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from multiprocessing import cpu_count
|
from multiprocessing import cpu_count
|
||||||
from pathlib import PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
from time import perf_counter
|
from time import perf_counter
|
||||||
from urllib.parse import unquote
|
from urllib.parse import unquote
|
||||||
from wsgiref.handlers import format_date_time
|
from wsgiref.handlers import format_date_time
|
||||||
|
|
||||||
import av
|
import httpx
|
||||||
import fitz # PyMuPDF
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import numpy as np
|
|
||||||
import pyvips
|
|
||||||
from blake3 import blake3
|
from blake3 import blake3
|
||||||
from sanic import Blueprint, empty, raw, redirect
|
from sanic import Blueprint, empty, raw, redirect
|
||||||
from sanic.exceptions import NotFound
|
from sanic.exceptions import NotFound
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
|
|
||||||
from cista import auth, config, sharefs
|
from cista import auth, config, onlyoffice, sharefs, watching
|
||||||
from cista.preview_worker import PreviewRequest, PreviewResponse
|
from cista.fileio import fuid
|
||||||
|
from cista.preview_worker import (
|
||||||
|
DOC_PREVIEW_SUFFIXES,
|
||||||
|
OFFICE_PREVIEW_SUFFIXES,
|
||||||
|
PreviewRequest,
|
||||||
|
PreviewResponse,
|
||||||
|
)
|
||||||
from cista.util.filename import sanitize
|
from cista.util.filename import sanitize
|
||||||
|
|
||||||
# OnlyOffice integration is loaded lazily; availability is checked at runtime.
|
|
||||||
_onlyoffice = None
|
|
||||||
|
|
||||||
|
|
||||||
def _get_onlyoffice():
|
|
||||||
global _onlyoffice
|
|
||||||
if _onlyoffice is None:
|
|
||||||
try:
|
|
||||||
from cista import onlyoffice as oo
|
|
||||||
|
|
||||||
_onlyoffice = oo
|
|
||||||
except Exception:
|
|
||||||
_onlyoffice = False
|
|
||||||
return _onlyoffice
|
|
||||||
|
|
||||||
|
|
||||||
bp = Blueprint("preview", url_prefix="/preview")
|
bp = Blueprint("preview", url_prefix="/preview")
|
||||||
|
|
||||||
|
|
||||||
@@ -90,7 +75,7 @@ class PreviewCache:
|
|||||||
# Global preview cache instance
|
# Global preview cache instance
|
||||||
_preview_cache = PreviewCache(capacity=500)
|
_preview_cache = PreviewCache(capacity=500)
|
||||||
|
|
||||||
PREVIEW_TIMEOUT = 3.0 # seconds until preview subprocess is killed
|
PREVIEW_TIMEOUT = 10.0 # seconds until preview subprocess is killed
|
||||||
PREVIEW_WORKERS = max(2, min(8, cpu_count()))
|
PREVIEW_WORKERS = max(2, min(8, cpu_count()))
|
||||||
_active_procs: set[asyncio.subprocess.Process] = set()
|
_active_procs: set[asyncio.subprocess.Process] = set()
|
||||||
_preview_pool = None
|
_preview_pool = None
|
||||||
@@ -112,14 +97,20 @@ class _PreviewWorker:
|
|||||||
def __init__(self, proc: asyncio.subprocess.Process):
|
def __init__(self, proc: asyncio.subprocess.Process):
|
||||||
self.proc = proc
|
self.proc = proc
|
||||||
|
|
||||||
async def request(self, filepath, quality: int, maxsize: int, maxzoom: float):
|
async def request(
|
||||||
|
self,
|
||||||
|
filepath,
|
||||||
|
quality: int,
|
||||||
|
maxsize: int,
|
||||||
|
maxzoom: float,
|
||||||
|
data: bytes | None = None,
|
||||||
|
):
|
||||||
if self.proc.returncode is not None:
|
if self.proc.returncode is not None:
|
||||||
raise WorkerProtocolError("worker already exited")
|
raise WorkerProtocolError("worker already exited")
|
||||||
if self.proc.stdin is None or self.proc.stdout is None:
|
if self.proc.stdin is None or self.proc.stdout is None:
|
||||||
raise WorkerProtocolError("worker streams not available")
|
raise WorkerProtocolError("worker streams not available")
|
||||||
|
|
||||||
line = (
|
meta = msgspec.json.encode(
|
||||||
msgspec.json.encode(
|
|
||||||
PreviewRequest(
|
PreviewRequest(
|
||||||
path=str(filepath),
|
path=str(filepath),
|
||||||
quality=quality,
|
quality=quality,
|
||||||
@@ -127,9 +118,9 @@ class _PreviewWorker:
|
|||||||
maxzoom=maxzoom,
|
maxzoom=maxzoom,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
+ b"\n"
|
payload = data or b""
|
||||||
)
|
packet = struct.pack("<II", len(meta), len(payload)) + meta + payload
|
||||||
self.proc.stdin.write(line)
|
self.proc.stdin.write(packet)
|
||||||
await self.proc.stdin.drain()
|
await self.proc.stdin.drain()
|
||||||
|
|
||||||
checksum = await self.proc.stdout.readexactly(WORKER_CHECKSUM_BYTES)
|
checksum = await self.proc.stdout.readexactly(WORKER_CHECKSUM_BYTES)
|
||||||
@@ -164,9 +155,22 @@ class _PreviewWorkerPool:
|
|||||||
def __init__(self, size: int):
|
def __init__(self, size: int):
|
||||||
self.size = size
|
self.size = size
|
||||||
self._idle: asyncio.Queue[_PreviewWorker] = asyncio.Queue()
|
self._idle: asyncio.Queue[_PreviewWorker] = asyncio.Queue()
|
||||||
|
self._pending: asyncio.PriorityQueue[tuple[int, int, asyncio.Future, tuple]] = (
|
||||||
|
asyncio.PriorityQueue()
|
||||||
|
)
|
||||||
self._workers: set[_PreviewWorker] = set()
|
self._workers: set[_PreviewWorker] = set()
|
||||||
|
self._dispatchers: list[asyncio.Task] = []
|
||||||
|
self._seq = 0
|
||||||
self._closed = False
|
self._closed = False
|
||||||
|
|
||||||
|
async def _read_startup_stderr(self, proc: asyncio.subprocess.Process) -> str:
|
||||||
|
if proc.stderr is None:
|
||||||
|
return ""
|
||||||
|
with contextlib.suppress(TimeoutError):
|
||||||
|
data = await asyncio.wait_for(proc.stderr.read(), timeout=0.5)
|
||||||
|
return data.decode(errors="replace").strip()
|
||||||
|
return ""
|
||||||
|
|
||||||
async def _spawn_worker(self) -> _PreviewWorker:
|
async def _spawn_worker(self) -> _PreviewWorker:
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
sys.executable,
|
sys.executable,
|
||||||
@@ -174,10 +178,35 @@ class _PreviewWorkerPool:
|
|||||||
"cista.preview_worker",
|
"cista.preview_worker",
|
||||||
stdin=asyncio.subprocess.PIPE,
|
stdin=asyncio.subprocess.PIPE,
|
||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
stderr=asyncio.subprocess.DEVNULL,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
start_new_session=True,
|
start_new_session=True,
|
||||||
)
|
)
|
||||||
_active_procs.add(proc)
|
_active_procs.add(proc)
|
||||||
|
try:
|
||||||
|
ready = await asyncio.wait_for(proc.stdout.readexactly(1), timeout=30.0)
|
||||||
|
except TimeoutError as err:
|
||||||
|
with contextlib.suppress(ProcessLookupError):
|
||||||
|
proc.kill()
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
await proc.wait()
|
||||||
|
stderr = await self._read_startup_stderr(proc)
|
||||||
|
if stderr:
|
||||||
|
raise WorkerProtocolError(
|
||||||
|
"preview worker failed to become ready: " + stderr.splitlines()[-1]
|
||||||
|
) from err
|
||||||
|
raise WorkerProtocolError("preview worker failed to become ready") from err
|
||||||
|
except asyncio.IncompleteReadError as err:
|
||||||
|
stderr = await self._read_startup_stderr(proc)
|
||||||
|
if stderr:
|
||||||
|
raise WorkerProtocolError(
|
||||||
|
"preview worker exited before signalling readiness: "
|
||||||
|
+ stderr.splitlines()[-1]
|
||||||
|
) from err
|
||||||
|
raise WorkerProtocolError(
|
||||||
|
"preview worker exited before signalling readiness"
|
||||||
|
) from err
|
||||||
|
if ready != b"\x01":
|
||||||
|
raise WorkerProtocolError(f"preview worker ready signal invalid: {ready!r}")
|
||||||
return _PreviewWorker(proc)
|
return _PreviewWorker(proc)
|
||||||
|
|
||||||
async def _add_worker(self) -> None:
|
async def _add_worker(self) -> None:
|
||||||
@@ -195,33 +224,63 @@ class _PreviewWorkerPool:
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Failed to replace preview worker")
|
logger.exception("Failed to replace preview worker")
|
||||||
|
|
||||||
async def start(self) -> None:
|
async def _dispatch_loop(self) -> None:
|
||||||
for _ in range(self.size):
|
while True:
|
||||||
await self._add_worker()
|
try:
|
||||||
|
_priority, _seq, future, args = await self._pending.get()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
return
|
||||||
|
|
||||||
async def run(self, filepath, quality: int, maxsize: int, maxzoom: float):
|
if future.cancelled():
|
||||||
if self._closed:
|
continue
|
||||||
raise PreviewError("preview worker pool closed")
|
|
||||||
worker = await self._idle.get()
|
try:
|
||||||
|
worker = await asyncio.wait_for(
|
||||||
|
self._idle.get(), timeout=PREVIEW_TIMEOUT
|
||||||
|
)
|
||||||
|
except TimeoutError:
|
||||||
|
logger.warning(
|
||||||
|
"Preview worker unavailable (%ds) for %s",
|
||||||
|
int(PREVIEW_TIMEOUT),
|
||||||
|
args[0].name,
|
||||||
|
)
|
||||||
|
if not future.done():
|
||||||
|
future.set_exception(
|
||||||
|
PreviewTimeoutError(
|
||||||
|
args[0].name,
|
||||||
|
backend=_expected_preview_backend(args[0]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
filepath = args[0]
|
||||||
replace = False
|
replace = False
|
||||||
try:
|
try:
|
||||||
out, resp = await asyncio.wait_for(
|
out, resp = await asyncio.wait_for(
|
||||||
worker.request(filepath, quality, maxsize, maxzoom),
|
worker.request(*args),
|
||||||
timeout=PREVIEW_TIMEOUT,
|
timeout=PREVIEW_TIMEOUT,
|
||||||
)
|
)
|
||||||
return out, resp
|
if not future.done():
|
||||||
|
future.set_result((out, resp))
|
||||||
except TimeoutError:
|
except TimeoutError:
|
||||||
replace = True
|
replace = True
|
||||||
logger.warning(
|
if not future.done():
|
||||||
"Preview timeout (%ds) for %s", int(PREVIEW_TIMEOUT), filepath.name
|
future.set_exception(
|
||||||
|
PreviewTimeoutError(
|
||||||
|
filepath.name,
|
||||||
|
backend=_expected_preview_backend(filepath),
|
||||||
)
|
)
|
||||||
raise PreviewTimeoutError(filepath.name) from None
|
)
|
||||||
except WorkerChecksumError as e:
|
except WorkerChecksumError:
|
||||||
replace = True
|
replace = True
|
||||||
logger.error("Preview checksum mismatch for %s", filepath.name)
|
logger.error("Preview checksum mismatch for %s", filepath.name)
|
||||||
raise PreviewError(f"worker checksum mismatch for {filepath.name}") from e
|
if not future.done():
|
||||||
except PreviewError:
|
future.set_exception(
|
||||||
raise
|
PreviewError(f"worker checksum mismatch for {filepath.name}")
|
||||||
|
)
|
||||||
|
except PreviewError as e:
|
||||||
|
if not future.done():
|
||||||
|
future.set_exception(e)
|
||||||
except (
|
except (
|
||||||
WorkerProtocolError,
|
WorkerProtocolError,
|
||||||
asyncio.IncompleteReadError,
|
asyncio.IncompleteReadError,
|
||||||
@@ -235,9 +294,21 @@ class _PreviewWorkerPool:
|
|||||||
logger.warning(
|
logger.warning(
|
||||||
"Preview worker protocol failure for %s: %s", filepath.name, e
|
"Preview worker protocol failure for %s: %s", filepath.name, e
|
||||||
)
|
)
|
||||||
raise PreviewError(
|
if not future.done():
|
||||||
|
future.set_exception(
|
||||||
|
PreviewError(
|
||||||
f"worker protocol failure for {filepath.name}: {e}"
|
f"worker protocol failure for {filepath.name}: {e}"
|
||||||
) from e
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
replace = True
|
||||||
|
logger.exception(
|
||||||
|
"Unexpected preview worker error for %s", filepath.name
|
||||||
|
)
|
||||||
|
if not future.done():
|
||||||
|
future.set_exception(
|
||||||
|
PreviewError(f"unexpected worker error for {filepath.name}")
|
||||||
|
)
|
||||||
finally:
|
finally:
|
||||||
if replace:
|
if replace:
|
||||||
await self._replace_worker(worker)
|
await self._replace_worker(worker)
|
||||||
@@ -246,10 +317,55 @@ class _PreviewWorkerPool:
|
|||||||
else:
|
else:
|
||||||
await self._replace_worker(worker)
|
await self._replace_worker(worker)
|
||||||
|
|
||||||
|
async def start(self) -> None:
|
||||||
|
workers = await asyncio.gather(
|
||||||
|
*(self._spawn_worker() for _ in range(self.size))
|
||||||
|
)
|
||||||
|
for worker in workers:
|
||||||
|
self._workers.add(worker)
|
||||||
|
await self._idle.put(worker)
|
||||||
|
for _ in range(self.size):
|
||||||
|
self._dispatchers.append(asyncio.create_task(self._dispatch_loop()))
|
||||||
|
|
||||||
|
async def run(
|
||||||
|
self,
|
||||||
|
filepath,
|
||||||
|
quality: int,
|
||||||
|
maxsize: int,
|
||||||
|
maxzoom: float,
|
||||||
|
data: bytes | None = None,
|
||||||
|
):
|
||||||
|
if self._closed:
|
||||||
|
raise PreviewError("preview worker pool closed")
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
future = loop.create_future()
|
||||||
|
self._seq += 1
|
||||||
|
await self._pending.put(
|
||||||
|
(
|
||||||
|
_preview_job_priority(filepath),
|
||||||
|
self._seq,
|
||||||
|
future,
|
||||||
|
(filepath, quality, maxsize, maxzoom, data),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return await future
|
||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self) -> None:
|
||||||
self._closed = True
|
self._closed = True
|
||||||
|
for task in self._dispatchers:
|
||||||
|
task.cancel()
|
||||||
|
if self._dispatchers:
|
||||||
|
await asyncio.gather(*self._dispatchers, return_exceptions=True)
|
||||||
|
self._dispatchers.clear()
|
||||||
workers = list(self._workers)
|
workers = list(self._workers)
|
||||||
self._workers.clear()
|
self._workers.clear()
|
||||||
|
while not self._pending.empty():
|
||||||
|
try:
|
||||||
|
_priority, _seq, future, _args = self._pending.get_nowait()
|
||||||
|
except asyncio.QueueEmpty:
|
||||||
|
break
|
||||||
|
if not future.done():
|
||||||
|
future.set_exception(PreviewError("preview worker pool closed"))
|
||||||
while not self._idle.empty():
|
while not self._idle.empty():
|
||||||
try:
|
try:
|
||||||
self._idle.get_nowait()
|
self._idle.get_nowait()
|
||||||
@@ -302,6 +418,10 @@ async def verify_preview(request):
|
|||||||
class PreviewTimeoutError(Exception):
|
class PreviewTimeoutError(Exception):
|
||||||
"""Raised when the preview subprocess exceeds PREVIEW_TIMEOUT."""
|
"""Raised when the preview subprocess exceeds PREVIEW_TIMEOUT."""
|
||||||
|
|
||||||
|
def __init__(self, message: str, *, backend: str | None = None):
|
||||||
|
super().__init__(message)
|
||||||
|
self.backend = backend
|
||||||
|
|
||||||
|
|
||||||
class PreviewError(Exception):
|
class PreviewError(Exception):
|
||||||
"""Raised when the preview subprocess exits with a non-zero status."""
|
"""Raised when the preview subprocess exits with a non-zero status."""
|
||||||
@@ -318,58 +438,143 @@ class PreviewError(Exception):
|
|||||||
self.backend = backend
|
self.backend = backend
|
||||||
|
|
||||||
|
|
||||||
|
# Max concurrent OnlyOffice conversion requests. OO has its own queue;
|
||||||
|
# we must not flood it. This is intentionally small.
|
||||||
|
OO_MAX_CONCURRENT = PREVIEW_WORKERS
|
||||||
|
|
||||||
|
|
||||||
|
class OOConversionManager:
|
||||||
|
"""Manages async OnlyOffice conversions with deduplication and concurrency limits."""
|
||||||
|
|
||||||
|
def __init__(self, max_concurrent: int = OO_MAX_CONCURRENT):
|
||||||
|
self._semaphore = asyncio.Semaphore(max_concurrent)
|
||||||
|
self._in_flight: dict[str, asyncio.Future[bytes]] = {}
|
||||||
|
self._tasks: set[asyncio.Task[None]] = set()
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
|
async def convert(self, filepath: Path) -> bytes:
|
||||||
|
"""Return PNG bytes for *filepath*, deduplicating concurrent requests."""
|
||||||
|
stat = await asyncio.to_thread(filepath.stat)
|
||||||
|
key = f"{filepath}:{stat.st_mtime_ns}"
|
||||||
|
|
||||||
|
async with self._lock:
|
||||||
|
if key in self._in_flight:
|
||||||
|
future = self._in_flight[key]
|
||||||
|
else:
|
||||||
|
future = asyncio.get_running_loop().create_future()
|
||||||
|
self._in_flight[key] = future
|
||||||
|
task = asyncio.create_task(self._do_convert(filepath, key, future))
|
||||||
|
self._tasks.add(task)
|
||||||
|
task.add_done_callback(self._tasks.discard)
|
||||||
|
|
||||||
|
return await future
|
||||||
|
|
||||||
|
async def _do_convert(
|
||||||
|
self, filepath: Path, key: str, future: asyncio.Future[bytes]
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
async with self._semaphore:
|
||||||
|
png_bytes = await onlyoffice.convert_to_png_async(
|
||||||
|
filepath, request_timeout=5.0
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
if not future.done():
|
||||||
|
future.set_exception(e)
|
||||||
|
async with self._lock:
|
||||||
|
self._in_flight.pop(key, None)
|
||||||
|
else:
|
||||||
|
if not future.done():
|
||||||
|
future.set_result(png_bytes)
|
||||||
|
async with self._lock:
|
||||||
|
self._in_flight.pop(key, None)
|
||||||
|
|
||||||
|
|
||||||
|
_oo_manager: OOConversionManager | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def get_oo_manager() -> OOConversionManager:
|
||||||
|
"""Return the singleton OOConversionManager."""
|
||||||
|
global _oo_manager
|
||||||
|
if _oo_manager is None:
|
||||||
|
_oo_manager = OOConversionManager(max_concurrent=OO_MAX_CONCURRENT)
|
||||||
|
return _oo_manager
|
||||||
|
|
||||||
|
|
||||||
|
async def _generate_office_preview(
|
||||||
|
filepath: Path, quality: int, maxsize: int, maxzoom: float
|
||||||
|
) -> tuple[bytes | None, PreviewResponse | None]:
|
||||||
|
"""Generate a preview for an office file using OnlyOffice + worker AVIF conversion."""
|
||||||
|
manager = get_oo_manager()
|
||||||
|
t_oo_start = perf_counter()
|
||||||
|
png_bytes = await manager.convert(filepath)
|
||||||
|
t_oo_end = perf_counter()
|
||||||
|
|
||||||
|
img, resp = await _run_preview_process(
|
||||||
|
filepath, quality, maxsize, maxzoom, data=png_bytes
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp is not None:
|
||||||
|
resp.backend = "onlyoffice+" + (resp.backend or "pyvips")
|
||||||
|
if resp.timings:
|
||||||
|
resp.timings = [round((t_oo_end - t_oo_start) * 1000, 1), *resp.timings]
|
||||||
|
return img, resp
|
||||||
|
|
||||||
|
|
||||||
async def _run_preview_process(
|
async def _run_preview_process(
|
||||||
filepath, quality: int, maxsize: int, maxzoom: float
|
filepath, quality: int, maxsize: int, maxzoom: float, data: bytes | None = None
|
||||||
) -> tuple[bytes | None, PreviewResponse | None]:
|
) -> tuple[bytes | None, PreviewResponse | None]:
|
||||||
"""Run preview request in a persistent worker process."""
|
"""Run preview request in a persistent worker process."""
|
||||||
await start_preview_workers()
|
await start_preview_workers()
|
||||||
if _preview_pool is None:
|
if _preview_pool is None:
|
||||||
raise PreviewError(f"preview worker pool unavailable for {filepath.name}")
|
raise PreviewError(f"preview worker pool unavailable for {filepath.name}")
|
||||||
return await _preview_pool.run(filepath, quality, maxsize, maxzoom)
|
return await _preview_pool.run(filepath, quality, maxsize, maxzoom, data)
|
||||||
|
|
||||||
|
|
||||||
DOC_PREVIEW_SUFFIXES = {".pdf", ".xps", ".epub", ".mobi"}
|
def _onlyoffice_error_short_text(detail: str) -> str:
|
||||||
|
if detail.startswith("OnlyOffice conversion error:"):
|
||||||
|
code = detail.rsplit(":", 1)[-1].strip()
|
||||||
|
return {
|
||||||
|
"-8": "onlyoffice jwt error",
|
||||||
|
"-4": "onlyoffice input error",
|
||||||
|
"-2": "onlyoffice timeout error",
|
||||||
|
"-1": "onlyoffice unknown error",
|
||||||
|
}.get(code, f"onlyoffice {code} error")
|
||||||
|
if "OnlyOffice response did not contain FileUrl" in detail:
|
||||||
|
return "onlyoffice no-fileurl error"
|
||||||
|
return "onlyoffice error"
|
||||||
|
|
||||||
OFFICE_PREVIEW_SUFFIXES = {
|
|
||||||
".doc",
|
def _preview_job_priority(path) -> int:
|
||||||
".dot",
|
"""Return priority for preview job (lower=higher priority).
|
||||||
".docx",
|
|
||||||
".docm",
|
Priority order: images (0) < video (1) < PDF (2) < office (3) < unknown (4)
|
||||||
".dotx",
|
"""
|
||||||
".dotm",
|
suffix = path.suffix.lower()
|
||||||
".rtf",
|
if suffix in DOC_PREVIEW_SUFFIXES:
|
||||||
".odt",
|
return 2
|
||||||
".ott",
|
if suffix in OFFICE_PREVIEW_SUFFIXES:
|
||||||
".txt",
|
return 3
|
||||||
".md",
|
mime_type, _ = mimetypes.guess_type(path.name)
|
||||||
".mhtml",
|
if mime_type and mime_type.startswith("image/"):
|
||||||
".mht",
|
return 0
|
||||||
".html",
|
if mime_type and mime_type.startswith("video/"):
|
||||||
".htm",
|
return 1
|
||||||
".xml",
|
return 4
|
||||||
".wps",
|
|
||||||
".wri",
|
|
||||||
# Spreadsheets
|
def _expected_preview_backend(path: Path) -> str:
|
||||||
".xls",
|
"""Best-effort backend label used for timeout/access logging."""
|
||||||
".xlsx",
|
suffix = path.suffix.lower()
|
||||||
".xlsm",
|
if suffix in OFFICE_PREVIEW_SUFFIXES:
|
||||||
".xlsb",
|
return "onlyoffice"
|
||||||
".xltx",
|
if suffix in DOC_PREVIEW_SUFFIXES:
|
||||||
".xltm",
|
return "pdf"
|
||||||
".ods",
|
mime_type, _ = mimetypes.guess_type(path.name)
|
||||||
".ots",
|
if mime_type and mime_type.startswith("video/"):
|
||||||
".csv",
|
return "video"
|
||||||
# Presentations
|
if mime_type and mime_type.startswith("image/"):
|
||||||
".ppt",
|
return "pyvips"
|
||||||
".pptx",
|
return "preview"
|
||||||
".pptm",
|
|
||||||
".pps",
|
|
||||||
".ppsx",
|
|
||||||
".pot",
|
|
||||||
".potx",
|
|
||||||
".odp",
|
|
||||||
".otp",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def is_previewable_path(path) -> bool:
|
def is_previewable_path(path) -> bool:
|
||||||
@@ -422,14 +627,39 @@ async def preview(req, path):
|
|||||||
|
|
||||||
# Generate preview
|
# Generate preview
|
||||||
try:
|
try:
|
||||||
img, preview_resp = await _run_preview_process(
|
if filepath.suffix.lower() in OFFICE_PREVIEW_SUFFIXES:
|
||||||
filepath, quality, maxsize, maxzoom
|
img, preview_resp = await asyncio.wait_for(
|
||||||
|
_generate_office_preview(filepath, quality, maxsize, maxzoom),
|
||||||
|
timeout=PREVIEW_TIMEOUT,
|
||||||
)
|
)
|
||||||
except PreviewTimeoutError:
|
else:
|
||||||
return empty(504)
|
img, preview_resp = await asyncio.wait_for(
|
||||||
|
_run_preview_process(filepath, quality, maxsize, maxzoom),
|
||||||
|
timeout=PREVIEW_TIMEOUT,
|
||||||
|
)
|
||||||
|
except TimeoutError:
|
||||||
|
req.ctx.log_extra = f"{_expected_preview_backend(filepath)} timeout"
|
||||||
|
return empty(503)
|
||||||
|
except PreviewTimeoutError as e:
|
||||||
|
req.ctx.log_extra = (
|
||||||
|
f"{(e.backend or _expected_preview_backend(filepath))} timeout"
|
||||||
|
)
|
||||||
|
return empty(503)
|
||||||
|
except httpx.HTTPStatusError:
|
||||||
|
req.ctx.log_extra = "onlyoffice N/A"
|
||||||
|
return empty(503)
|
||||||
|
except httpx.RequestError:
|
||||||
|
req.ctx.log_extra = "onlyoffice N/A"
|
||||||
|
return empty(503)
|
||||||
|
except RuntimeError as e:
|
||||||
|
detail = str(e)
|
||||||
|
if detail.startswith("OnlyOffice"):
|
||||||
|
req.ctx.log_extra = _onlyoffice_error_short_text(detail)
|
||||||
|
return empty(503)
|
||||||
|
raise
|
||||||
except PreviewError as e:
|
except PreviewError as e:
|
||||||
if e.backend:
|
if e.backend:
|
||||||
req.ctx._log_extra = e.backend
|
req.ctx.log_extra = e.backend
|
||||||
detail = str(e)
|
detail = str(e)
|
||||||
if detail == "preview worker error" and e.stderr:
|
if detail == "preview worker error" and e.stderr:
|
||||||
captured = e.stderr.strip()
|
captured = e.stderr.strip()
|
||||||
@@ -437,18 +667,30 @@ async def preview(req, path):
|
|||||||
detail = captured.splitlines()[0]
|
detail = captured.splitlines()[0]
|
||||||
logger.error("%s preview: %s", filepath, detail)
|
logger.error("%s preview: %s", filepath, detail)
|
||||||
return empty(422)
|
return empty(422)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
req.ctx.log_extra = "preview cancelled"
|
||||||
|
return empty(503)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Unhandled preview error for %s", filepath)
|
||||||
|
return empty(500)
|
||||||
if preview_resp and preview_resp.backend:
|
if preview_resp and preview_resp.backend:
|
||||||
if preview_resp.timings:
|
if preview_resp.timings:
|
||||||
timing_detail = "/".join(
|
timing_detail = "/".join(
|
||||||
str(round(value)) for value in preview_resp.timings
|
str(round(value)) for value in preview_resp.timings
|
||||||
)
|
)
|
||||||
req.ctx._log_extra = f"{preview_resp.backend} {timing_detail} ➛"
|
req.ctx.log_extra = f"{preview_resp.backend} {timing_detail} ➛"
|
||||||
else:
|
else:
|
||||||
req.ctx._log_extra = preview_resp.backend
|
req.ctx.log_extra = preview_resp.backend
|
||||||
if not img:
|
if not img:
|
||||||
# Preview generation failed, redirect to the file itself
|
# Preview generation failed, redirect to the file itself
|
||||||
return redirect(f"/files/{path}", status=303)
|
return redirect(f"/files/{path}", status=303)
|
||||||
|
|
||||||
|
# Store aspect ratio if the worker returned dimensions
|
||||||
|
if preview_resp and preview_resp.width and preview_resp.height:
|
||||||
|
ar = round(preview_resp.height / preview_resp.width, 2)
|
||||||
|
fuid_str = fuid(stat)
|
||||||
|
watching.notify_ar(fuid_str, ar)
|
||||||
|
|
||||||
# Build headers and cache the full response
|
# Build headers and cache the full response
|
||||||
preview_mime = (
|
preview_mime = (
|
||||||
preview_resp.mime
|
preview_resp.mime
|
||||||
@@ -467,240 +709,3 @@ async def preview(req, path):
|
|||||||
_preview_cache.set(etag, CachedPreview(headers=headers, body=img))
|
_preview_cache.set(etag, CachedPreview(headers=headers, body=img))
|
||||||
|
|
||||||
return raw(img, headers=headers)
|
return raw(img, headers=headers)
|
||||||
|
|
||||||
|
|
||||||
def dispatch(path, quality, maxsize, maxzoom):
|
|
||||||
backend = "unknown"
|
|
||||||
try:
|
|
||||||
suffix = path.suffix.lower()
|
|
||||||
if suffix in DOC_PREVIEW_SUFFIXES:
|
|
||||||
backend = "pdf"
|
|
||||||
return process_pdf(path, quality=quality, maxsize=maxsize, maxzoom=maxzoom)
|
|
||||||
if suffix in OFFICE_PREVIEW_SUFFIXES:
|
|
||||||
backend = "onlyoffice"
|
|
||||||
return process_office(
|
|
||||||
path, quality=quality, maxsize=maxsize, maxzoom=maxzoom
|
|
||||||
)
|
|
||||||
mime_type, _ = mimetypes.guess_type(path.name)
|
|
||||||
if mime_type and mime_type.startswith("video/"):
|
|
||||||
backend = "video"
|
|
||||||
return process_video(path, quality=quality, maxsize=maxsize)
|
|
||||||
if mime_type and mime_type.startswith("image/"):
|
|
||||||
backend = "pyvips"
|
|
||||||
return process_image(path, quality=quality, maxsize=maxsize)
|
|
||||||
except ValueError as e:
|
|
||||||
return None, PreviewResponse(ok=False, backend=backend, error=str(e))
|
|
||||||
except Exception as e:
|
|
||||||
return None, PreviewResponse(ok=False, backend=backend, error=str(e))
|
|
||||||
return None, PreviewResponse(ok=False, backend=backend, error="preview unsupported")
|
|
||||||
|
|
||||||
|
|
||||||
def process_image(path, *, maxsize, quality):
|
|
||||||
return process_image_pyvips(path, maxsize=maxsize, quality=quality)
|
|
||||||
|
|
||||||
|
|
||||||
def process_image_pyvips(path, *, maxsize, quality):
|
|
||||||
t_start = perf_counter()
|
|
||||||
img = pyvips.Image.new_from_file(str(path), access="sequential")
|
|
||||||
img = img.autorot()
|
|
||||||
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
|
|
||||||
if scale < 1.0:
|
|
||||||
img = img.resize(scale)
|
|
||||||
ret = img.write_to_buffer(
|
|
||||||
".avif",
|
|
||||||
Q=quality,
|
|
||||||
effort=AVIF_FAST_EFFORT,
|
|
||||||
strip=True,
|
|
||||||
)
|
|
||||||
t_end = perf_counter()
|
|
||||||
|
|
||||||
return ret, PreviewResponse(
|
|
||||||
ok=True,
|
|
||||||
mime="image/avif",
|
|
||||||
backend="pyvips",
|
|
||||||
timings=[round((t_end - t_start) * 1000, 1)],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
|
||||||
t_load_start = perf_counter()
|
|
||||||
pdf = fitz.open(path)
|
|
||||||
page = pdf.load_page(page_number)
|
|
||||||
w, h = page.rect[2:4]
|
|
||||||
zoom = min(maxsize / w, maxsize / h, maxzoom)
|
|
||||||
mat = fitz.Matrix(zoom, zoom)
|
|
||||||
pix = page.get_pixmap(matrix=mat)
|
|
||||||
t_load_end = perf_counter()
|
|
||||||
|
|
||||||
t_save_start = perf_counter()
|
|
||||||
img = pyvips.Image.new_from_memory(
|
|
||||||
pix.samples_mv, pix.width, pix.height, pix.n, "uchar"
|
|
||||||
)
|
|
||||||
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True)
|
|
||||||
backend = "pdf+pyvips"
|
|
||||||
t_save_end = perf_counter()
|
|
||||||
|
|
||||||
return ret, PreviewResponse(
|
|
||||||
ok=True,
|
|
||||||
mime="image/avif",
|
|
||||||
backend=backend,
|
|
||||||
timings=[
|
|
||||||
round((t_load_end - t_load_start) * 1000, 1),
|
|
||||||
round((t_save_end - t_save_start) * 1000, 1),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def process_office(path, *, quality, maxsize, maxzoom):
|
|
||||||
t_load_start = perf_counter()
|
|
||||||
oo = _get_onlyoffice()
|
|
||||||
if oo is False:
|
|
||||||
raise RuntimeError("OnlyOffice is not installed")
|
|
||||||
if not oo.is_available():
|
|
||||||
raise RuntimeError("OnlyOffice Document Server is not reachable")
|
|
||||||
png_bytes = oo.convert_to_png(path)
|
|
||||||
t_load_end = perf_counter()
|
|
||||||
|
|
||||||
t_save_start = perf_counter()
|
|
||||||
img = pyvips.Image.new_from_buffer(png_bytes, "")
|
|
||||||
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
|
|
||||||
if scale < 1.0:
|
|
||||||
img = img.resize(scale)
|
|
||||||
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True)
|
|
||||||
backend = "onlyoffice+pyvips"
|
|
||||||
t_save_end = perf_counter()
|
|
||||||
|
|
||||||
return ret, PreviewResponse(
|
|
||||||
ok=True,
|
|
||||||
mime="image/avif",
|
|
||||||
backend=backend,
|
|
||||||
timings=[
|
|
||||||
round((t_load_end - t_load_start) * 1000, 1),
|
|
||||||
round((t_save_end - t_save_start) * 1000, 1),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def process_video(path, *, maxsize, quality):
|
|
||||||
frame = None
|
|
||||||
imgdata = io.BytesIO()
|
|
||||||
istream = ostream = icc = occ = frame = None
|
|
||||||
t_load_start = perf_counter()
|
|
||||||
# Initialize to avoid "possibly unbound" in static analysis when exceptions occur
|
|
||||||
t_load_end = t_load_start
|
|
||||||
t_save_start = t_load_start
|
|
||||||
t_save_end = t_load_start
|
|
||||||
with (
|
|
||||||
av.open(
|
|
||||||
str(path),
|
|
||||||
options={
|
|
||||||
"analyzeduration": "1000000", # 1 second (in microseconds)
|
|
||||||
"fflags": "fastseek",
|
|
||||||
},
|
|
||||||
) as icontainer,
|
|
||||||
av.open(imgdata, "w", format="avif") as ocontainer,
|
|
||||||
):
|
|
||||||
istream = icontainer.streams.video[0]
|
|
||||||
istream.codec_context.skip_frame = "NONKEY"
|
|
||||||
icontainer.seek((icontainer.duration or 0) // 8)
|
|
||||||
for frame in icontainer.decode(istream):
|
|
||||||
if frame.dts is not None:
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
raise RuntimeError("No frames found in video")
|
|
||||||
|
|
||||||
# Resize frame to thumbnail size
|
|
||||||
if frame.width > maxsize or frame.height > maxsize:
|
|
||||||
scale_factor = min(maxsize / frame.width, maxsize / frame.height)
|
|
||||||
new_width = int(frame.width * scale_factor)
|
|
||||||
new_height = int(frame.height * scale_factor)
|
|
||||||
frame = frame.reformat(width=new_width, height=new_height)
|
|
||||||
|
|
||||||
# Apply EXIF rotation if present
|
|
||||||
if frame.rotation:
|
|
||||||
# frame.rotation indicates clockwise rotation needed to display correctly
|
|
||||||
# np.rot90 rotates counter-clockwise, so we negate k
|
|
||||||
k = (frame.rotation // 90) % 4 # Convert to counter-clockwise rotations
|
|
||||||
if k == 2:
|
|
||||||
# 180° rotation can be done in YUV420p, preserving HDR
|
|
||||||
try:
|
|
||||||
fplanes = frame.to_ndarray()
|
|
||||||
# Split into Y, U, V planes of proper dimensions
|
|
||||||
planes = [
|
|
||||||
fplanes[: frame.height],
|
|
||||||
fplanes[
|
|
||||||
frame.height : frame.height + frame.height // 4
|
|
||||||
].reshape(frame.height // 2, frame.width // 2),
|
|
||||||
fplanes[frame.height + frame.height // 4 :].reshape(
|
|
||||||
frame.height // 2, frame.width // 2
|
|
||||||
),
|
|
||||||
]
|
|
||||||
# Rotate each plane by 180°
|
|
||||||
planes = [np.rot90(p, 2) for p in planes]
|
|
||||||
# Restore PyAV format
|
|
||||||
planes = np.hstack([p.flat for p in planes]).reshape(
|
|
||||||
-1, planes[0].shape[1]
|
|
||||||
)
|
|
||||||
frame = av.VideoFrame.from_ndarray(planes, format=frame.format.name)
|
|
||||||
del planes, fplanes
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(f"Error rotating video frame by 180°: {e}")
|
|
||||||
elif k in (1, 3):
|
|
||||||
# 90° or 270° rotation requires RGB conversion (loses HDR)
|
|
||||||
try:
|
|
||||||
rgb = frame.to_ndarray(format="rgb24")
|
|
||||||
rgb = np.rot90(rgb, k)
|
|
||||||
frame = av.VideoFrame.from_ndarray(rgb, format="rgb24")
|
|
||||||
frame = frame.reformat(
|
|
||||||
format="yuv420p"
|
|
||||||
) # Convert back for encoding
|
|
||||||
del rgb
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(
|
|
||||||
f"Error rotating video frame by {frame.rotation}°: {e}"
|
|
||||||
)
|
|
||||||
t_load_end = perf_counter()
|
|
||||||
|
|
||||||
t_save_start = perf_counter()
|
|
||||||
crf = str(int(63 * (1 - quality / 100) ** 2)) # Closely matching PIL quality-%
|
|
||||||
ostream = ocontainer.add_stream(
|
|
||||||
"av1",
|
|
||||||
options={
|
|
||||||
"crf": crf,
|
|
||||||
"usage": "realtime",
|
|
||||||
"cpu-used": "8",
|
|
||||||
"threads": "1",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if not isinstance(ostream, av.VideoStream):
|
|
||||||
raise PreviewError("failed to initialize AV1 video stream")
|
|
||||||
ostream.width = frame.width
|
|
||||||
ostream.height = frame.height
|
|
||||||
ostream.pix_fmt = frame.format.name
|
|
||||||
icc = istream.codec_context
|
|
||||||
occ = ostream.codec_context
|
|
||||||
|
|
||||||
# Copy HDR metadata from input video stream
|
|
||||||
occ.color_primaries = icc.color_primaries
|
|
||||||
occ.color_trc = icc.color_trc
|
|
||||||
occ.colorspace = icc.colorspace
|
|
||||||
occ.color_range = icc.color_range
|
|
||||||
|
|
||||||
ocontainer.mux(ostream.encode(frame))
|
|
||||||
ocontainer.mux(ostream.encode(None)) # Flush the stream
|
|
||||||
t_save_end = perf_counter()
|
|
||||||
|
|
||||||
# Capture result before cleanup
|
|
||||||
ret = imgdata.getvalue()
|
|
||||||
resp = PreviewResponse(
|
|
||||||
ok=True,
|
|
||||||
mime="image/avif",
|
|
||||||
backend="video",
|
|
||||||
timings=[
|
|
||||||
round((t_load_end - t_load_start) * 1000, 1),
|
|
||||||
round((t_save_end - t_save_start) * 1000, 1),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
del imgdata, istream, ostream, icc, occ, frame
|
|
||||||
gc.collect()
|
|
||||||
return ret, resp
|
|
||||||
|
|||||||
+444
-11
@@ -1,24 +1,85 @@
|
|||||||
"""Preview generation worker subprocess.
|
"""Preview generation worker subprocess and synchronous preview engine.
|
||||||
|
|
||||||
Two modes are supported:
|
Two modes are supported:
|
||||||
1) Legacy one-shot mode: argv has path/quality/maxsize/maxzoom.
|
1) Legacy one-shot mode: argv has path/quality/maxsize/maxzoom.
|
||||||
2) Long-lived mode: read JSONL commands from stdin and write framed responses.
|
2) Long-lived mode: read framed requests from stdin and write framed responses.
|
||||||
|
|
||||||
Framed response format:
|
Framed request format (stdin):
|
||||||
|
(uint32 json size)(uint32 data size)(json)(binary data)
|
||||||
|
|
||||||
|
Framed response format (stdout):
|
||||||
(blake3(packet))(uint32 json size)(uint32 payload size)(json)(binary payload)
|
(blake3(packet))(uint32 json size)(uint32 payload size)(json)(binary payload)
|
||||||
where packet = (uint32 json size)(uint32 payload size)(json)(binary payload).
|
where packet = (uint32 json size)(uint32 payload size)(json)(binary payload).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
|
import gc
|
||||||
import io
|
import io
|
||||||
import logging
|
import logging
|
||||||
|
import mimetypes
|
||||||
import struct
|
import struct
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from time import perf_counter
|
||||||
|
|
||||||
|
import av
|
||||||
|
import fitz # PyMuPDF
|
||||||
import msgspec
|
import msgspec
|
||||||
|
import numpy as np
|
||||||
|
import pyvips
|
||||||
from blake3 import blake3
|
from blake3 import blake3
|
||||||
|
|
||||||
|
from cista import config
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
AVIF_FAST_EFFORT = 0
|
||||||
|
|
||||||
|
DOC_PREVIEW_SUFFIXES = {".pdf", ".xps", ".epub", ".mobi"}
|
||||||
|
|
||||||
|
OFFICE_PREVIEW_SUFFIXES = {
|
||||||
|
".doc",
|
||||||
|
".dot",
|
||||||
|
".docx",
|
||||||
|
".docm",
|
||||||
|
".dotx",
|
||||||
|
".dotm",
|
||||||
|
".rtf",
|
||||||
|
".odt",
|
||||||
|
".ott",
|
||||||
|
".txt",
|
||||||
|
".md",
|
||||||
|
".mhtml",
|
||||||
|
".mht",
|
||||||
|
".html",
|
||||||
|
".htm",
|
||||||
|
".xml",
|
||||||
|
".wps",
|
||||||
|
".wri",
|
||||||
|
# Spreadsheets
|
||||||
|
".xls",
|
||||||
|
".xlsx",
|
||||||
|
".xlsm",
|
||||||
|
".xlsb",
|
||||||
|
".xltx",
|
||||||
|
".xltm",
|
||||||
|
".ods",
|
||||||
|
".ots",
|
||||||
|
".csv",
|
||||||
|
# Presentations
|
||||||
|
".ppt",
|
||||||
|
".pptx",
|
||||||
|
".pptm",
|
||||||
|
".pps",
|
||||||
|
".ppsx",
|
||||||
|
".pot",
|
||||||
|
".potx",
|
||||||
|
".odp",
|
||||||
|
".otp",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class PreviewRequest(msgspec.Struct, omit_defaults=True):
|
class PreviewRequest(msgspec.Struct, omit_defaults=True):
|
||||||
path: str
|
path: str
|
||||||
@@ -34,12 +95,38 @@ class PreviewResponse(msgspec.Struct, omit_defaults=True):
|
|||||||
timings: list[float] | None = None
|
timings: list[float] | None = None
|
||||||
error: str | None = None
|
error: str | None = None
|
||||||
stderr: str | None = None
|
stderr: str | None = None
|
||||||
|
width: int | None = None
|
||||||
|
height: int | None = None
|
||||||
|
|
||||||
|
|
||||||
_enc = msgspec.json.Encoder()
|
_enc = msgspec.json.Encoder()
|
||||||
_dec_req = msgspec.json.Decoder(PreviewRequest)
|
_dec_req = msgspec.json.Decoder(PreviewRequest)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_exactly(f, n: int) -> bytes:
|
||||||
|
buf = b""
|
||||||
|
while len(buf) < n:
|
||||||
|
chunk = f.read(n - len(buf))
|
||||||
|
if not chunk:
|
||||||
|
raise EOFError
|
||||||
|
buf += chunk
|
||||||
|
return buf
|
||||||
|
|
||||||
|
|
||||||
|
def _read_request() -> tuple[PreviewRequest, bytes] | None:
|
||||||
|
try:
|
||||||
|
header = _read_exactly(sys.stdin.buffer, 8)
|
||||||
|
except EOFError:
|
||||||
|
return None
|
||||||
|
json_size, data_size = struct.unpack("<II", header)
|
||||||
|
meta_raw = _read_exactly(sys.stdin.buffer, json_size)
|
||||||
|
data = b""
|
||||||
|
if data_size:
|
||||||
|
data = _read_exactly(sys.stdin.buffer, data_size)
|
||||||
|
req = _dec_req.decode(meta_raw)
|
||||||
|
return req, data
|
||||||
|
|
||||||
|
|
||||||
def _write_response(resp: PreviewResponse, payload: bytes) -> None:
|
def _write_response(resp: PreviewResponse, payload: bytes) -> None:
|
||||||
meta_bytes = _enc.encode(resp)
|
meta_bytes = _enc.encode(resp)
|
||||||
packet = struct.pack("<II", len(meta_bytes), len(payload)) + meta_bytes + payload
|
packet = struct.pack("<II", len(meta_bytes), len(payload)) + meta_bytes + payload
|
||||||
@@ -49,13 +136,347 @@ def _write_response(resp: PreviewResponse, payload: bytes) -> None:
|
|||||||
sys.stdout.buffer.flush()
|
sys.stdout.buffer.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def dispatch(path, quality, maxsize, maxzoom, data=None):
|
||||||
|
backend = "unknown"
|
||||||
|
try:
|
||||||
|
if data:
|
||||||
|
backend = "pyvips"
|
||||||
|
return process_image_buffer(
|
||||||
|
data, quality=quality, maxsize=maxsize, maxzoom=maxzoom
|
||||||
|
)
|
||||||
|
suffix = path.suffix.lower()
|
||||||
|
if suffix in DOC_PREVIEW_SUFFIXES:
|
||||||
|
backend = "pdf"
|
||||||
|
return process_pdf(path, quality=quality, maxsize=maxsize, maxzoom=maxzoom)
|
||||||
|
mime_type, _ = mimetypes.guess_type(path.name)
|
||||||
|
if mime_type and mime_type.startswith("video/"):
|
||||||
|
backend = "video"
|
||||||
|
return process_video(path, quality=quality, maxsize=maxsize)
|
||||||
|
if mime_type and mime_type.startswith("image/"):
|
||||||
|
backend = "pyvips"
|
||||||
|
return process_image(path, quality=quality, maxsize=maxsize)
|
||||||
|
except ValueError as e:
|
||||||
|
return None, PreviewResponse(ok=False, backend=backend, error=str(e))
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Preview dispatch failed for %s", path)
|
||||||
|
return None, PreviewResponse(ok=False, backend=backend, error=str(e))
|
||||||
|
return None, PreviewResponse(ok=False, backend=backend, error="preview unsupported")
|
||||||
|
|
||||||
|
|
||||||
|
def process_image(path, *, maxsize, quality):
|
||||||
|
return process_image_pyvips(path, maxsize=maxsize, quality=quality)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_image_dimensions(path: Path) -> tuple[int, int] | None:
|
||||||
|
"""Probe image dimensions.
|
||||||
|
|
||||||
|
pyvips can read the header of most formats (including HEIC) without
|
||||||
|
fully decoding the image.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
img = pyvips.Image.new_from_file(str(path))
|
||||||
|
img = img.autorot()
|
||||||
|
except pyvips.error.Error:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return img.width, img.height
|
||||||
|
|
||||||
|
|
||||||
|
def _image_via_ffmpeg(path: Path, maxsize: int, quality: int) -> bytes:
|
||||||
|
"""Convert any image to AVIF using ffmpeg CLI.
|
||||||
|
|
||||||
|
ffmpeg handles HEIC tile assembly, EXIF rotation, HDR metadata and
|
||||||
|
ICC profile embedding automatically.
|
||||||
|
"""
|
||||||
|
dims = _get_image_dimensions(path)
|
||||||
|
crf = int(63 * (1 - quality / 100) ** 2)
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".avif", delete=False) as tmp_f:
|
||||||
|
tmp_path = tmp_f.name
|
||||||
|
cmd = [
|
||||||
|
"ffmpeg",
|
||||||
|
"-y",
|
||||||
|
"-i",
|
||||||
|
str(path),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-c:v",
|
||||||
|
"av1",
|
||||||
|
"-crf",
|
||||||
|
str(crf),
|
||||||
|
"-cpu-used",
|
||||||
|
"8",
|
||||||
|
tmp_path,
|
||||||
|
]
|
||||||
|
if dims is not None:
|
||||||
|
w, h = dims
|
||||||
|
if max(w, h) > maxsize:
|
||||||
|
scale = min(maxsize / w, maxsize / h)
|
||||||
|
new_w = int(w * scale)
|
||||||
|
new_h = int(h * scale)
|
||||||
|
# insert -s <wxh> right after the input file
|
||||||
|
cmd.insert(4, "-s")
|
||||||
|
cmd.insert(5, f"{new_w}x{new_h}")
|
||||||
|
try:
|
||||||
|
subprocess.run(cmd, capture_output=True, check=True, shell=False) # noqa: S603
|
||||||
|
with Path(tmp_path).open("rb") as f:
|
||||||
|
return f.read()
|
||||||
|
finally:
|
||||||
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def process_image_pyvips(path, *, maxsize, quality):
|
||||||
|
t_start = perf_counter()
|
||||||
|
suffix = path.suffix.lower()
|
||||||
|
|
||||||
|
# HEIC/HEIF: ffmpeg handles tile assembly and HDR correctly;
|
||||||
|
# skip pyvips entirely.
|
||||||
|
if suffix in (".heic", ".heif"):
|
||||||
|
heic_dims = _get_image_dimensions(path)
|
||||||
|
width, height = heic_dims or (None, None)
|
||||||
|
ret = _image_via_ffmpeg(path, maxsize, quality)
|
||||||
|
t_end = perf_counter()
|
||||||
|
return ret, PreviewResponse(
|
||||||
|
ok=True,
|
||||||
|
mime="image/avif",
|
||||||
|
backend="ffmpeg",
|
||||||
|
timings=[round((t_end - t_start) * 1000, 1)],
|
||||||
|
width=width,
|
||||||
|
height=height,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Other image formats: pyvips first, ffmpeg fallback.
|
||||||
|
load_opts = {"access": "sequential"}
|
||||||
|
orig_w = orig_h = None
|
||||||
|
try:
|
||||||
|
img = pyvips.Image.new_from_file(str(path), **load_opts)
|
||||||
|
img = img.autorot()
|
||||||
|
orig_w, orig_h = img.width, img.height
|
||||||
|
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
|
||||||
|
if scale < 1.0:
|
||||||
|
img = img.resize(scale)
|
||||||
|
ret = img.write_to_buffer(
|
||||||
|
".avif",
|
||||||
|
Q=quality,
|
||||||
|
effort=AVIF_FAST_EFFORT,
|
||||||
|
strip=True,
|
||||||
|
)
|
||||||
|
backend = "pyvips"
|
||||||
|
except pyvips.error.Error:
|
||||||
|
orig_w, orig_h = None, None
|
||||||
|
ret = _image_via_ffmpeg(path, maxsize, quality)
|
||||||
|
backend = "ffmpeg"
|
||||||
|
t_end = perf_counter()
|
||||||
|
|
||||||
|
return ret, PreviewResponse(
|
||||||
|
ok=True,
|
||||||
|
mime="image/avif",
|
||||||
|
backend=backend,
|
||||||
|
timings=[round((t_end - t_start) * 1000, 1)],
|
||||||
|
width=orig_w,
|
||||||
|
height=orig_h,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def process_image_buffer(data: bytes, *, quality, maxsize, maxzoom):
|
||||||
|
_ = maxzoom
|
||||||
|
t_start = perf_counter()
|
||||||
|
img = pyvips.Image.new_from_buffer(data, "")
|
||||||
|
img = img.autorot()
|
||||||
|
orig_w, orig_h = img.width, img.height
|
||||||
|
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
|
||||||
|
if scale < 1.0:
|
||||||
|
img = img.resize(scale)
|
||||||
|
ret = img.write_to_buffer(
|
||||||
|
".avif",
|
||||||
|
Q=quality,
|
||||||
|
effort=AVIF_FAST_EFFORT,
|
||||||
|
strip=True,
|
||||||
|
)
|
||||||
|
t_end = perf_counter()
|
||||||
|
|
||||||
|
return ret, PreviewResponse(
|
||||||
|
ok=True,
|
||||||
|
mime="image/avif",
|
||||||
|
backend="pyvips",
|
||||||
|
timings=[round((t_end - t_start) * 1000, 1)],
|
||||||
|
width=orig_w,
|
||||||
|
height=orig_h,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
|
||||||
|
t_load_start = perf_counter()
|
||||||
|
pdf = fitz.open(path)
|
||||||
|
page = pdf.load_page(page_number)
|
||||||
|
w, h = page.rect[2:4]
|
||||||
|
zoom = min(maxsize / w, maxsize / h, maxzoom)
|
||||||
|
mat = fitz.Matrix(zoom, zoom)
|
||||||
|
pix = page.get_pixmap(matrix=mat)
|
||||||
|
t_load_end = perf_counter()
|
||||||
|
|
||||||
|
t_save_start = perf_counter()
|
||||||
|
img = pyvips.Image.new_from_memory(
|
||||||
|
pix.samples_mv, pix.width, pix.height, pix.n, "uchar"
|
||||||
|
)
|
||||||
|
ret = img.write_to_buffer(".avif", Q=quality, effort=AVIF_FAST_EFFORT, strip=True)
|
||||||
|
backend = "pdf+pyvips"
|
||||||
|
t_save_end = perf_counter()
|
||||||
|
|
||||||
|
return ret, PreviewResponse(
|
||||||
|
ok=True,
|
||||||
|
mime="image/avif",
|
||||||
|
backend=backend,
|
||||||
|
timings=[
|
||||||
|
round((t_load_end - t_load_start) * 1000, 1),
|
||||||
|
round((t_save_end - t_save_start) * 1000, 1),
|
||||||
|
],
|
||||||
|
width=round(w),
|
||||||
|
height=round(h),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def process_video(path, *, maxsize, quality):
|
||||||
|
frame = None
|
||||||
|
imgdata = io.BytesIO()
|
||||||
|
istream = ostream = icc = occ = frame = None
|
||||||
|
t_load_start = perf_counter()
|
||||||
|
# Initialize to avoid "possibly unbound" in static analysis when exceptions occur
|
||||||
|
t_load_end = t_load_start
|
||||||
|
t_save_start = t_load_start
|
||||||
|
t_save_end = t_load_start
|
||||||
|
with (
|
||||||
|
av.open(
|
||||||
|
str(path),
|
||||||
|
options={
|
||||||
|
"analyzeduration": "1000000", # 1 second (in microseconds)
|
||||||
|
"fflags": "fastseek",
|
||||||
|
},
|
||||||
|
) as icontainer,
|
||||||
|
av.open(imgdata, "w", format="avif") as ocontainer,
|
||||||
|
):
|
||||||
|
istream = icontainer.streams.video[0]
|
||||||
|
istream.codec_context.skip_frame = "NONKEY"
|
||||||
|
icontainer.seek((icontainer.duration or 0) // 8)
|
||||||
|
for frame in icontainer.decode(istream):
|
||||||
|
if frame.dts is not None:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise RuntimeError("No frames found in video")
|
||||||
|
|
||||||
|
# Resize frame to thumbnail size
|
||||||
|
# Capture display dimensions before resize (accounting for rotation)
|
||||||
|
disp_w = frame.width
|
||||||
|
disp_h = frame.height
|
||||||
|
if frame.rotation in (90, 270):
|
||||||
|
disp_w, disp_h = disp_h, disp_w
|
||||||
|
if frame.width > maxsize or frame.height > maxsize:
|
||||||
|
scale_factor = min(maxsize / frame.width, maxsize / frame.height)
|
||||||
|
new_width = int(frame.width * scale_factor)
|
||||||
|
new_height = int(frame.height * scale_factor)
|
||||||
|
frame = frame.reformat(width=new_width, height=new_height)
|
||||||
|
|
||||||
|
# Apply EXIF rotation if present
|
||||||
|
if frame.rotation:
|
||||||
|
# frame.rotation indicates clockwise rotation needed to display correctly
|
||||||
|
# np.rot90 rotates counter-clockwise, so we negate k
|
||||||
|
k = (frame.rotation // 90) % 4 # Convert to counter-clockwise rotations
|
||||||
|
if k == 2:
|
||||||
|
# 180° rotation can be done in YUV420p, preserving HDR
|
||||||
|
try:
|
||||||
|
fplanes = frame.to_ndarray()
|
||||||
|
# Split into Y, U, V planes of proper dimensions
|
||||||
|
planes = [
|
||||||
|
fplanes[: frame.height],
|
||||||
|
fplanes[
|
||||||
|
frame.height : frame.height + frame.height // 4
|
||||||
|
].reshape(frame.height // 2, frame.width // 2),
|
||||||
|
fplanes[frame.height + frame.height // 4 :].reshape(
|
||||||
|
frame.height // 2, frame.width // 2
|
||||||
|
),
|
||||||
|
]
|
||||||
|
# Rotate each plane by 180°
|
||||||
|
planes = [np.rot90(p, 2) for p in planes]
|
||||||
|
# Restore PyAV format
|
||||||
|
planes = np.hstack([p.flat for p in planes]).reshape(
|
||||||
|
-1, planes[0].shape[1]
|
||||||
|
)
|
||||||
|
frame = av.VideoFrame.from_ndarray(planes, format=frame.format.name)
|
||||||
|
del planes, fplanes
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Error rotating video frame by 180°")
|
||||||
|
elif k in (1, 3):
|
||||||
|
# 90° or 270° rotation requires RGB conversion (loses HDR)
|
||||||
|
try:
|
||||||
|
rgb = frame.to_ndarray(format="rgb24")
|
||||||
|
rgb = np.rot90(rgb, k)
|
||||||
|
frame = av.VideoFrame.from_ndarray(rgb, format="rgb24")
|
||||||
|
frame = frame.reformat(
|
||||||
|
format="yuv420p"
|
||||||
|
) # Convert back for encoding
|
||||||
|
del rgb
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"Error rotating video frame by %s°", frame.rotation
|
||||||
|
)
|
||||||
|
|
||||||
|
# libsvtav1 rejects full-range JPEG-style YUV pixel formats such as
|
||||||
|
# yuvj420p, so normalize them before opening the encoder.
|
||||||
|
if frame.format.name.startswith("yuvj"):
|
||||||
|
frame = frame.reformat(format="yuv420p")
|
||||||
|
t_load_end = perf_counter()
|
||||||
|
|
||||||
|
t_save_start = perf_counter()
|
||||||
|
crf = str(int(63 * (1 - quality / 100) ** 2)) # Closely matching PIL quality-%
|
||||||
|
ostream = ocontainer.add_stream(
|
||||||
|
"av1",
|
||||||
|
options={
|
||||||
|
"crf": crf,
|
||||||
|
"usage": "realtime",
|
||||||
|
"cpu-used": "8",
|
||||||
|
"threads": "1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not isinstance(ostream, av.VideoStream):
|
||||||
|
raise TypeError("failed to initialize AV1 video stream")
|
||||||
|
ostream.width = frame.width
|
||||||
|
ostream.height = frame.height
|
||||||
|
ostream.pix_fmt = frame.format.name
|
||||||
|
icc = istream.codec_context
|
||||||
|
occ = ostream.codec_context
|
||||||
|
|
||||||
|
# Copy HDR metadata from input video stream
|
||||||
|
occ.color_primaries = icc.color_primaries
|
||||||
|
occ.color_trc = icc.color_trc
|
||||||
|
occ.colorspace = icc.colorspace
|
||||||
|
occ.color_range = icc.color_range
|
||||||
|
|
||||||
|
ocontainer.mux(ostream.encode(frame))
|
||||||
|
ocontainer.mux(ostream.encode(None)) # Flush the stream
|
||||||
|
t_save_end = perf_counter()
|
||||||
|
|
||||||
|
# Capture result before cleanup
|
||||||
|
ret = imgdata.getvalue()
|
||||||
|
resp = PreviewResponse(
|
||||||
|
ok=True,
|
||||||
|
mime="image/avif",
|
||||||
|
backend="video",
|
||||||
|
timings=[
|
||||||
|
round((t_load_end - t_load_start) * 1000, 1),
|
||||||
|
round((t_save_end - t_save_start) * 1000, 1),
|
||||||
|
],
|
||||||
|
width=disp_w,
|
||||||
|
height=disp_h,
|
||||||
|
)
|
||||||
|
del imgdata, istream, ostream, icc, occ, frame
|
||||||
|
gc.collect()
|
||||||
|
return ret, resp
|
||||||
|
|
||||||
|
|
||||||
def _run_once() -> None:
|
def _run_once() -> None:
|
||||||
if len(sys.argv) != 5:
|
if len(sys.argv) != 5:
|
||||||
sys.stderr.write(f"Usage: {sys.argv[0]} <path> <quality> <maxsize> <maxzoom>\n")
|
sys.stderr.write(f"Usage: {sys.argv[0]} <path> <quality> <maxsize> <maxzoom>\n")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
from cista.preview import dispatch
|
|
||||||
|
|
||||||
path = Path(sys.argv[1])
|
path = Path(sys.argv[1])
|
||||||
quality = int(sys.argv[2])
|
quality = int(sys.argv[2])
|
||||||
maxsize = int(sys.argv[3])
|
maxsize = int(sys.argv[3])
|
||||||
@@ -67,21 +488,19 @@ def _run_once() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _run_loop() -> None:
|
def _run_loop() -> None:
|
||||||
from cista.preview import dispatch
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
line = sys.stdin.buffer.readline()
|
result = _read_request()
|
||||||
if not line:
|
if result is None:
|
||||||
return
|
return
|
||||||
|
req, data = result
|
||||||
stderr_capture = io.StringIO()
|
stderr_capture = io.StringIO()
|
||||||
handler = logging.StreamHandler(stderr_capture)
|
handler = logging.StreamHandler(stderr_capture)
|
||||||
root_logger = logging.getLogger()
|
root_logger = logging.getLogger()
|
||||||
root_logger.addHandler(handler)
|
root_logger.addHandler(handler)
|
||||||
try:
|
try:
|
||||||
with contextlib.redirect_stderr(stderr_capture):
|
with contextlib.redirect_stderr(stderr_capture):
|
||||||
req = _dec_req.decode(line)
|
|
||||||
result, resp = dispatch(
|
result, resp = dispatch(
|
||||||
Path(req.path), req.quality, req.maxsize, req.maxzoom
|
Path(req.path), req.quality, req.maxsize, req.maxzoom, data
|
||||||
)
|
)
|
||||||
if not resp.ok:
|
if not resp.ok:
|
||||||
captured = stderr_capture.getvalue().strip()
|
captured = stderr_capture.getvalue().strip()
|
||||||
@@ -94,6 +513,7 @@ def _run_loop() -> None:
|
|||||||
)
|
)
|
||||||
_write_response(resp, result or b"")
|
_write_response(resp, result or b"")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
logger.exception("Preview worker error for %s", req.path)
|
||||||
captured = stderr_capture.getvalue().strip()
|
captured = stderr_capture.getvalue().strip()
|
||||||
_write_response(
|
_write_response(
|
||||||
PreviewResponse(ok=False, error=str(e), stderr=captured or None), b""
|
PreviewResponse(ok=False, error=str(e), stderr=captured or None), b""
|
||||||
@@ -106,9 +526,22 @@ def _run_loop() -> None:
|
|||||||
def main() -> None:
|
def main() -> None:
|
||||||
# Configure all log output to stderr before any imports that may emit logs.
|
# Configure all log output to stderr before any imports that may emit logs.
|
||||||
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
|
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
|
||||||
|
try:
|
||||||
|
config.load_config()
|
||||||
|
logger.warning(
|
||||||
|
"preview-worker config=%s master_secret=%s",
|
||||||
|
config.conffile,
|
||||||
|
config.config.secret,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("preview-worker failed to load config at startup")
|
||||||
if len(sys.argv) > 1:
|
if len(sys.argv) > 1:
|
||||||
_run_once()
|
_run_once()
|
||||||
return
|
return
|
||||||
|
# Eagerly import heavy modules before signalling readiness so the parent
|
||||||
|
# does not hand us a request while we are still initialising.
|
||||||
|
sys.stdout.buffer.write(b"\x01")
|
||||||
|
sys.stdout.buffer.flush()
|
||||||
_run_loop()
|
_run_loop()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+2
-1
@@ -12,7 +12,7 @@ class ErrorMsg(msgspec.Struct):
|
|||||||
## Directory listings
|
## Directory listings
|
||||||
|
|
||||||
|
|
||||||
class FileEntry(msgspec.Struct, array_like=True, frozen=True):
|
class FileEntry(msgspec.Struct, array_like=True, frozen=True, omit_defaults=True):
|
||||||
level: int
|
level: int
|
||||||
name: str
|
name: str
|
||||||
key: str
|
key: str
|
||||||
@@ -20,6 +20,7 @@ class FileEntry(msgspec.Struct, array_like=True, frozen=True):
|
|||||||
size: int
|
size: int
|
||||||
allocated: int
|
allocated: int
|
||||||
isfile: int
|
isfile: int
|
||||||
|
ar: float | None = None
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.key or "FileEntry()"
|
return self.key or "FileEntry()"
|
||||||
|
|||||||
+55
-11
@@ -1,12 +1,49 @@
|
|||||||
"""Custom access logging middleware for Sanic."""
|
"""Custom access logging middleware for Sanic."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unicodedata
|
import unicodedata
|
||||||
from ipaddress import IPv6Address
|
from ipaddress import IPv6Address
|
||||||
|
|
||||||
|
from sanic.log import LOGGING_CONFIG_DEFAULTS
|
||||||
|
|
||||||
logger = logging.getLogger("cista.access")
|
logger = logging.getLogger("cista.access")
|
||||||
|
|
||||||
|
|
||||||
|
class ReentrantSafeStreamHandler(logging.StreamHandler):
|
||||||
|
"""Stream handler that degrades gracefully on signal-time reentrant writes.
|
||||||
|
|
||||||
|
Python's buffered text streams are not reentrant. If a signal handler logs
|
||||||
|
while another log write is in progress, StreamHandler.emit can raise:
|
||||||
|
RuntimeError("reentrant call inside <_io.BufferedWriter ...>")
|
||||||
|
|
||||||
|
Instead of letting logging emit a long "--- Logging error ---" traceback,
|
||||||
|
we fall back to a best-effort os.write to the same file descriptor.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def emit(self, record: logging.LogRecord) -> None:
|
||||||
|
msg = ""
|
||||||
|
try:
|
||||||
|
msg = self.format(record)
|
||||||
|
stream = self.stream
|
||||||
|
stream.write(msg + self.terminator)
|
||||||
|
self.flush()
|
||||||
|
except RuntimeError as exc:
|
||||||
|
if "reentrant call inside" not in str(exc):
|
||||||
|
self.handleError(record)
|
||||||
|
return
|
||||||
|
stream = self.stream
|
||||||
|
fd = stream.fileno()
|
||||||
|
encoding = getattr(stream, "encoding", None) or "utf-8"
|
||||||
|
data = (msg + self.terminator).encode(encoding, errors="replace")
|
||||||
|
os.write(fd, data)
|
||||||
|
except RecursionError:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
self.handleError(record)
|
||||||
|
|
||||||
|
|
||||||
_RESET = "\033[0m"
|
_RESET = "\033[0m"
|
||||||
_STATUS_INFO = "\033[32m" # 1xx (green)
|
_STATUS_INFO = "\033[32m" # 1xx (green)
|
||||||
_STATUS_OK = "\033[1;92m" # 2xx (bright green)
|
_STATUS_OK = "\033[1;92m" # 2xx (bright green)
|
||||||
@@ -96,10 +133,11 @@ def format_duration_ms(duration_ms: float) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _display_width(text: str) -> int:
|
def _display_width(text: str) -> int:
|
||||||
width = 0
|
return sum(
|
||||||
for char in text:
|
1 + (unicodedata.east_asian_width(c) in "FW")
|
||||||
width += 2 if unicodedata.east_asian_width(char) in {"F", "W"} else 1
|
for c in text
|
||||||
return width
|
if unicodedata.category(c) != "Mn"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _format_left(label: str) -> str:
|
def _format_left(label: str) -> str:
|
||||||
@@ -233,7 +271,7 @@ def log_ws_close(
|
|||||||
|
|
||||||
def configure_access_logging() -> None:
|
def configure_access_logging() -> None:
|
||||||
"""Configure the cista.access logger to output to stderr."""
|
"""Configure the cista.access logger to output to stderr."""
|
||||||
handler = logging.StreamHandler(sys.stderr)
|
handler = ReentrantSafeStreamHandler(sys.stderr)
|
||||||
handler.setFormatter(logging.Formatter("%(message)s"))
|
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||||
logger.addHandler(handler)
|
logger.addHandler(handler)
|
||||||
logger.setLevel(logging.INFO)
|
logger.setLevel(logging.INFO)
|
||||||
@@ -242,20 +280,24 @@ def configure_access_logging() -> None:
|
|||||||
|
|
||||||
_LEVEL_EMOJI = {
|
_LEVEL_EMOJI = {
|
||||||
logging.DEBUG: "🔍",
|
logging.DEBUG: "🔍",
|
||||||
logging.INFO: "i",
|
logging.INFO: "ℹ️", # noqa: RUF001
|
||||||
logging.WARNING: "⚠️",
|
logging.WARNING: "⚠️",
|
||||||
logging.ERROR: "🛑",
|
logging.ERROR: "🛑",
|
||||||
logging.CRITICAL: "🛑",
|
logging.CRITICAL: "🛑",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _format_level_prefix(levelno: int) -> str:
|
||||||
|
emoji = _LEVEL_EMOJI.get(levelno, "▪️")
|
||||||
|
prefix = f"{emoji} "
|
||||||
|
return prefix + (" " * max(0, 3 - _display_width(prefix)))
|
||||||
|
|
||||||
|
|
||||||
class _EmojiFormatter(logging.Formatter):
|
class _EmojiFormatter(logging.Formatter):
|
||||||
"""Compact formatter: emoji + message, no timestamp/level text/logger name."""
|
"""Compact formatter: emoji + message, no timestamp/level text/logger name."""
|
||||||
|
|
||||||
def format(self, record: logging.LogRecord) -> str:
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
emoji = _LEVEL_EMOJI.get(record.levelno, "▪️")
|
return _format_level_prefix(record.levelno) + record.getMessage()
|
||||||
sep = " " if record.levelno in (logging.INFO, logging.WARNING) else " "
|
|
||||||
return f"{emoji}{sep}{record.getMessage()}"
|
|
||||||
|
|
||||||
|
|
||||||
def configure_main_logging() -> None:
|
def configure_main_logging() -> None:
|
||||||
@@ -264,8 +306,10 @@ def configure_main_logging() -> None:
|
|||||||
Patches LOGGING_CONFIG_DEFAULTS so the formatter survives every dictConfig
|
Patches LOGGING_CONFIG_DEFAULTS so the formatter survives every dictConfig
|
||||||
call Sanic makes during serve_single() / serve().
|
call Sanic makes during serve_single() / serve().
|
||||||
"""
|
"""
|
||||||
from sanic.log import LOGGING_CONFIG_DEFAULTS
|
for handler_name in ("console", "error_console", "access_console"):
|
||||||
|
LOGGING_CONFIG_DEFAULTS["handlers"][handler_name]["class"] = (
|
||||||
|
"cista.sanic_logging.ReentrantSafeStreamHandler"
|
||||||
|
)
|
||||||
LOGGING_CONFIG_DEFAULTS["formatters"]["generic"] = {
|
LOGGING_CONFIG_DEFAULTS["formatters"]["generic"] = {
|
||||||
"class": "cista.sanic_logging._EmojiFormatter",
|
"class": "cista.sanic_logging._EmojiFormatter",
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-3
@@ -4,14 +4,19 @@ from pathlib import Path
|
|||||||
|
|
||||||
from fastapi_vue.hostutil import parse_endpoint
|
from fastapi_vue.hostutil import parse_endpoint
|
||||||
from sanic import Sanic
|
from sanic import Sanic
|
||||||
|
from sanic.worker.loader import AppLoader
|
||||||
|
|
||||||
from cista import config, server80
|
from cista import config, server80
|
||||||
|
from cista.app import app
|
||||||
|
|
||||||
|
|
||||||
|
def load_app() -> Sanic:
|
||||||
|
"""Return the app instance for spawned Sanic worker/reloader processes."""
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
def run(*, dev=False):
|
def run(*, dev=False):
|
||||||
"""Run Sanic main process that spawns worker processes to serve HTTP requests."""
|
"""Run Sanic main process that spawns worker processes to serve HTTP requests."""
|
||||||
from .app import app
|
|
||||||
|
|
||||||
_url, opts = parse_listen(config.config.listen)
|
_url, opts = parse_listen(config.config.listen)
|
||||||
# Silence Sanic's warning about running in production rather than debug
|
# Silence Sanic's warning about running in production rather than debug
|
||||||
os.environ["SANIC_IGNORE_PRODUCTION_WARNING"] = "1"
|
os.environ["SANIC_IGNORE_PRODUCTION_WARNING"] = "1"
|
||||||
@@ -30,12 +35,13 @@ def run(*, dev=False):
|
|||||||
access_log=False,
|
access_log=False,
|
||||||
) # type: ignore[call-arg]
|
) # type: ignore[call-arg]
|
||||||
if dev:
|
if dev:
|
||||||
Sanic.serve()
|
Sanic.serve(app_loader=AppLoader(factory=load_app))
|
||||||
else:
|
else:
|
||||||
Sanic.serve_single()
|
Sanic.serve_single()
|
||||||
|
|
||||||
|
|
||||||
def check_cert(certdir, domain):
|
def check_cert(certdir, domain):
|
||||||
|
_ = domain
|
||||||
if (certdir / "privkey.pem").exist() and (certdir / "fullchain.pem").exists():
|
if (certdir / "privkey.pem").exist() and (certdir / "fullchain.pem").exists():
|
||||||
return
|
return
|
||||||
# Certificate provisioning is external; files must exist before startup.
|
# Certificate provisioning is external; files must exist before startup.
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ app = Sanic("server80")
|
|||||||
# Send all HTTP users to HTTPS
|
# Send all HTTP users to HTTPS
|
||||||
@app.exception(exceptions.NotFound, exceptions.MethodNotSupported)
|
@app.exception(exceptions.NotFound, exceptions.MethodNotSupported)
|
||||||
def redirect_everything_else(request, exception):
|
def redirect_everything_else(request, exception):
|
||||||
|
_ = exception
|
||||||
server, path = request.server_name, request.path
|
server, path = request.server_name, request.path
|
||||||
if server and path.startswith("/"):
|
if server and path.startswith("/"):
|
||||||
return response.redirect(f"https://{server}{path}", status=308)
|
return response.redirect(f"https://{server}{path}", status=308)
|
||||||
@@ -15,6 +16,7 @@ def redirect_everything_else(request, exception):
|
|||||||
# ACME challenge for LetsEncrypt
|
# ACME challenge for LetsEncrypt
|
||||||
@app.get("/.well-known/acme-challenge/<challenge>")
|
@app.get("/.well-known/acme-challenge/<challenge>")
|
||||||
async def letsencrypt(request, challenge):
|
async def letsencrypt(request, challenge):
|
||||||
|
_ = request
|
||||||
try:
|
try:
|
||||||
return response.text(acme_challenges[challenge])
|
return response.text(acme_challenges[challenge])
|
||||||
except KeyError:
|
except KeyError:
|
||||||
|
|||||||
+8
-1
@@ -36,7 +36,7 @@ def get(request):
|
|||||||
def create(request, res, username, **kwargs):
|
def create(request, res, username, **kwargs):
|
||||||
_purge_expired()
|
_purge_expired()
|
||||||
token = _token()
|
token = _token()
|
||||||
_sessions[token] = {"exp": int(time()) + max_age, "username": username, **kwargs}
|
put(token, username, **kwargs)
|
||||||
secure = request.scheme == "https"
|
secure = request.scheme == "https"
|
||||||
res.cookies.add_cookie(
|
res.cookies.add_cookie(
|
||||||
SESSION_COOKIE_NAME,
|
SESSION_COOKIE_NAME,
|
||||||
@@ -49,10 +49,17 @@ def create(request, res, username, **kwargs):
|
|||||||
|
|
||||||
|
|
||||||
def delete(request, res):
|
def delete(request, res):
|
||||||
|
token = request.cookies.get(SESSION_COOKIE_NAME)
|
||||||
|
if token is not None:
|
||||||
|
_sessions.pop(token, None)
|
||||||
secure = request.scheme == "https"
|
secure = request.scheme == "https"
|
||||||
res.cookies.delete_cookie(SESSION_COOKIE_NAME, host_prefix=secure)
|
res.cookies.delete_cookie(SESSION_COOKIE_NAME, host_prefix=secure)
|
||||||
|
|
||||||
|
|
||||||
|
def put(token: str, username: str, **kwargs) -> None:
|
||||||
|
_sessions[token] = {"exp": int(time()) + max_age, "username": username, **kwargs}
|
||||||
|
|
||||||
|
|
||||||
def flash(res, message: str | None):
|
def flash(res, message: str | None):
|
||||||
if message is None:
|
if message is None:
|
||||||
res.cookies.delete_cookie("message")
|
res.cookies.delete_cookie("message")
|
||||||
|
|||||||
+5
-2
@@ -107,10 +107,11 @@ async def validate_sso_request(request, *, perm: str = "cista:login") -> dict |
|
|||||||
request.ctx.sso_user = data
|
request.ctx.sso_user = data
|
||||||
if "set-cookie" in response.headers:
|
if "set-cookie" in response.headers:
|
||||||
request.ctx.sso_cookies = response.headers.get_list("set-cookie")
|
request.ctx.sso_cookies = response.headers.get_list("set-cookie")
|
||||||
return data
|
|
||||||
except Exception:
|
except Exception:
|
||||||
request.ctx.sso_user = {}
|
request.ctx.sso_user = {}
|
||||||
return {}
|
return {}
|
||||||
|
else:
|
||||||
|
return data
|
||||||
|
|
||||||
try:
|
try:
|
||||||
error_data = response.json()
|
error_data = response.json()
|
||||||
@@ -257,7 +258,7 @@ async def proxy_auth_request(request):
|
|||||||
method=request.method,
|
method=request.method,
|
||||||
url=url,
|
url=url,
|
||||||
headers=headers,
|
headers=headers,
|
||||||
content=request.body if request.body else None,
|
content=request.body or None,
|
||||||
) as response:
|
) as response:
|
||||||
raw_content = b"".join([chunk async for chunk in response.aiter_raw()])
|
raw_content = b"".join([chunk async for chunk in response.aiter_raw()])
|
||||||
|
|
||||||
@@ -348,6 +349,7 @@ bp = Blueprint("sso", url_prefix="/auth")
|
|||||||
@bp.websocket("/ws/<path:path>")
|
@bp.websocket("/ws/<path:path>")
|
||||||
async def auth_websocket_proxy(request, ws, path=""):
|
async def auth_websocket_proxy(request, ws, path=""):
|
||||||
"""Proxy WebSocket connections to the auth backend."""
|
"""Proxy WebSocket connections to the auth backend."""
|
||||||
|
_ = path
|
||||||
await proxy_auth_websocket(request, ws)
|
await proxy_auth_websocket(request, ws)
|
||||||
|
|
||||||
|
|
||||||
@@ -362,6 +364,7 @@ async def auth_websocket_proxy_root(request, ws):
|
|||||||
)
|
)
|
||||||
async def auth_proxy(request, path=""):
|
async def auth_proxy(request, path=""):
|
||||||
"""Proxy all auth requests to the auth backend."""
|
"""Proxy all auth requests to the auth backend."""
|
||||||
|
_ = path
|
||||||
return await proxy_auth_request(request)
|
return await proxy_auth_request(request)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import time
|
|||||||
from functools import wraps
|
from functools import wraps
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
|
import websockets.exceptions
|
||||||
from sanic import errorpages
|
from sanic import errorpages
|
||||||
from sanic.exceptions import SanicException
|
from sanic.exceptions import SanicException
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
@@ -60,13 +61,19 @@ def websocket_wrapper(handler):
|
|||||||
@wraps(handler)
|
@wraps(handler)
|
||||||
async def wrapper(request, ws, *args, **kwargs):
|
async def wrapper(request, ws, *args, **kwargs):
|
||||||
username = getattr(request.ctx, "username", None)
|
username = getattr(request.ctx, "username", None)
|
||||||
extra = username if username else None
|
extra = username or None
|
||||||
start = time.perf_counter()
|
start = time.perf_counter()
|
||||||
ws_id = log_ws_open(request, extra=extra)
|
ws_id = log_ws_open(request, extra=extra)
|
||||||
close_extra = None
|
close_extra = None
|
||||||
try:
|
try:
|
||||||
await auth.verify(request)
|
await auth.verify(request)
|
||||||
await handler(request, ws, *args, **kwargs)
|
await handler(request, ws, *args, **kwargs)
|
||||||
|
except (
|
||||||
|
websockets.exceptions.ConnectionClosedOK,
|
||||||
|
websockets.exceptions.ConnectionClosedError,
|
||||||
|
):
|
||||||
|
# Normal websocket closure - already logged in access log
|
||||||
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
context, code, message = {}, 500, str(e) or "Internal Server Error"
|
context, code, message = {}, 500, str(e) or "Internal Server Error"
|
||||||
if isinstance(e, SanicException):
|
if isinstance(e, SanicException):
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ class AsyncLink:
|
|||||||
@property
|
@property
|
||||||
def to_sync(self):
|
def to_sync(self):
|
||||||
"""Yield SyncRequests from async caller when called from worker thread."""
|
"""Yield SyncRequests from async caller when called from worker thread."""
|
||||||
while (req := self._await(self._get())) is not None:
|
while (req := self.await_sync(self._get())) is not None:
|
||||||
yield SyncRequest(self, req)
|
yield SyncRequest(self, req)
|
||||||
|
|
||||||
async def _get(self):
|
async def _get(self):
|
||||||
@@ -33,7 +33,7 @@ class AsyncLink:
|
|||||||
self.queue.task_done()
|
self.queue.task_done()
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def _await(self, coro):
|
def await_sync(self, coro):
|
||||||
"""Run coroutine in main thread and return result; called from worker."""
|
"""Run coroutine in main thread and return result; called from worker."""
|
||||||
return asyncio.run_coroutine_threadsafe(coro, self.loop).result()
|
return asyncio.run_coroutine_threadsafe(coro, self.loop).result()
|
||||||
|
|
||||||
@@ -87,9 +87,9 @@ class SyncRequest:
|
|||||||
def set_result(self, value):
|
def set_result(self, value):
|
||||||
"""Set result value; mark as done."""
|
"""Set result value; mark as done."""
|
||||||
self.done = True
|
self.done = True
|
||||||
self.alink._await(set_result(self.future, value))
|
self.alink.await_sync(set_result(self.future, value))
|
||||||
|
|
||||||
def set_exception(self, exc):
|
def set_exception(self, exc):
|
||||||
"""Set exception; mark as done."""
|
"""Set exception; mark as done."""
|
||||||
self.done = True
|
self.done = True
|
||||||
self.alink._await(set_result(self.future, exception=exc))
|
self.alink.await_sync(set_result(self.future, exception=exc))
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import hmac
|
||||||
|
import re
|
||||||
|
from typing import Protocol
|
||||||
|
from unicodedata import normalize
|
||||||
|
|
||||||
|
import argon2
|
||||||
|
|
||||||
|
_argon = argon2.PasswordHasher()
|
||||||
|
_droppyhash = re.compile(r"^([a-f0-9]{64})\$([a-f0-9]{8})$")
|
||||||
|
|
||||||
|
|
||||||
|
class SupportsHash(Protocol):
|
||||||
|
hash: str
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_secret(value: str) -> bytes:
|
||||||
|
return normalize("NFC", value).strip().encode()
|
||||||
|
|
||||||
|
|
||||||
|
def verify_hash(user_hash: str, *, username: str, password: str) -> bool:
|
||||||
|
"""Verify password hash and return whether the stored hash should be upgraded."""
|
||||||
|
if not user_hash:
|
||||||
|
raise ValueError("Account disabled")
|
||||||
|
|
||||||
|
normalized_username = normalize_secret(username)
|
||||||
|
normalized_password = normalize_secret(password)
|
||||||
|
|
||||||
|
if (match := _droppyhash.match(user_hash)) is not None:
|
||||||
|
expected_hash, salt = match.groups()
|
||||||
|
computed_hash = hmac.digest(
|
||||||
|
normalized_password + salt.encode() + normalized_username,
|
||||||
|
b"",
|
||||||
|
"sha256",
|
||||||
|
).hex()
|
||||||
|
if not hmac.compare_digest(expected_hash, computed_hash):
|
||||||
|
raise ValueError("Invalid password")
|
||||||
|
return True
|
||||||
|
|
||||||
|
try:
|
||||||
|
_argon.verify(user_hash, normalized_password)
|
||||||
|
except Exception:
|
||||||
|
raise ValueError("Invalid password") from None
|
||||||
|
return _argon.check_needs_rehash(user_hash)
|
||||||
|
|
||||||
|
|
||||||
|
def set_password(user: SupportsHash, password: str) -> None:
|
||||||
|
user.hash = _argon.hash(normalize_secret(password))
|
||||||
+78
-7
@@ -9,6 +9,7 @@ from os import stat_result
|
|||||||
from pathlib import Path, PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
from stat import S_ISDIR, S_ISREG
|
from stat import S_ISDIR, S_ISREG
|
||||||
|
|
||||||
|
import inotify.adapters
|
||||||
import msgspec
|
import msgspec
|
||||||
from natsort import humansorted, natsort_keygen, ns
|
from natsort import humansorted, natsort_keygen, ns
|
||||||
from sanic.log import logger
|
from sanic.log import logger
|
||||||
@@ -30,6 +31,7 @@ if sys.platform == "win32":
|
|||||||
|
|
||||||
def get_allocated_size(path: Path, st: stat_result) -> int:
|
def get_allocated_size(path: Path, st: stat_result) -> int:
|
||||||
"""Get actual disk allocation on Windows using GetCompressedFileSizeW."""
|
"""Get actual disk allocation on Windows using GetCompressedFileSizeW."""
|
||||||
|
_ = st
|
||||||
high = wintypes.DWORD()
|
high = wintypes.DWORD()
|
||||||
low = GetCompressedFileSizeW(str(path), ctypes.byref(high))
|
low = GetCompressedFileSizeW(str(path), ctypes.byref(high))
|
||||||
if low == INVALID_FILE_SIZE and ctypes.get_last_error() != 0:
|
if low == INVALID_FILE_SIZE and ctypes.get_last_error() != 0:
|
||||||
@@ -40,6 +42,7 @@ else:
|
|||||||
|
|
||||||
def get_allocated_size(path: Path, st: stat_result) -> int:
|
def get_allocated_size(path: Path, st: stat_result) -> int:
|
||||||
"""Get actual disk allocation on Unix using st_blocks."""
|
"""Get actual disk allocation on Unix using st_blocks."""
|
||||||
|
_ = path
|
||||||
# st_blocks is in 512-byte units
|
# st_blocks is in 512-byte units
|
||||||
return st.st_blocks * 512
|
return st.st_blocks * 512
|
||||||
|
|
||||||
@@ -48,6 +51,14 @@ pubsub = {}
|
|||||||
sortkey = natsort_keygen(alg=ns.LOCALE)
|
sortkey = natsort_keygen(alg=ns.LOCALE)
|
||||||
|
|
||||||
|
|
||||||
|
class FormatUpdateLoopError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _WatcherStoppingError(Exception):
|
||||||
|
"""Internal control-flow exception for quick watcher shutdown."""
|
||||||
|
|
||||||
|
|
||||||
class State:
|
class State:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.lock = threading.RLock()
|
self.lock = threading.RLock()
|
||||||
@@ -147,6 +158,17 @@ stop_event = threading.Event()
|
|||||||
# Thread-safe queue for signaling path updates from websockets
|
# Thread-safe queue for signaling path updates from websockets
|
||||||
_update_queue: queue.Queue[PurePosixPath] = queue.Queue()
|
_update_queue: queue.Queue[PurePosixPath] = queue.Queue()
|
||||||
|
|
||||||
|
# Thread-safe queue for AR updates from the preview worker
|
||||||
|
_ar_queue: queue.Queue[tuple[str, float]] = queue.Queue()
|
||||||
|
|
||||||
|
# AR map: fuid -> aspect ratio (height/width). Written only by the watcher thread.
|
||||||
|
_ar_map: dict[str, float] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def notify_ar(fuid_key: str, ar: float) -> None:
|
||||||
|
"""Called from preview handler to update the AR for a file."""
|
||||||
|
_ar_queue.put_nowait((fuid_key, ar))
|
||||||
|
|
||||||
|
|
||||||
def notify_change(*paths: PurePosixPath | str):
|
def notify_change(*paths: PurePosixPath | str):
|
||||||
"""Signal that paths have changed. Called from control/upload websockets."""
|
"""Signal that paths have changed. Called from control/upload websockets."""
|
||||||
@@ -179,14 +201,16 @@ def walk(rel: PurePosixPath, stat: stat_result | None = None) -> list[FileEntry]
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(f"get_allocated_size failed for {path}")
|
logger.exception(f"get_allocated_size failed for {path}")
|
||||||
allocated = st.st_size if isfile else 0
|
allocated = st.st_size if isfile else 0
|
||||||
|
key = fuid(st)
|
||||||
entry = FileEntry(
|
entry = FileEntry(
|
||||||
level=len(rel.parts),
|
level=len(rel.parts),
|
||||||
name=rel.name,
|
name=rel.name,
|
||||||
key=fuid(st),
|
key=key,
|
||||||
mtime=int(st.st_mtime),
|
mtime=int(st.st_mtime),
|
||||||
size=st.st_size if isfile else 0,
|
size=st.st_size if isfile else 0,
|
||||||
allocated=allocated,
|
allocated=allocated,
|
||||||
isfile=isfile,
|
isfile=isfile,
|
||||||
|
ar=_ar_map.get(key) if isfile else None,
|
||||||
)
|
)
|
||||||
if isfile:
|
if isfile:
|
||||||
return [entry]
|
return [entry]
|
||||||
@@ -195,7 +219,7 @@ def walk(rel: PurePosixPath, stat: stat_result | None = None) -> list[FileEntry]
|
|||||||
li = []
|
li = []
|
||||||
for f in path.iterdir():
|
for f in path.iterdir():
|
||||||
if stop_event.is_set():
|
if stop_event.is_set():
|
||||||
raise SystemExit("quit")
|
raise _WatcherStoppingError
|
||||||
if f.name.startswith("."):
|
if f.name.startswith("."):
|
||||||
continue # No dotfiles
|
continue # No dotfiles
|
||||||
with suppress(FileNotFoundError):
|
with suppress(FileNotFoundError):
|
||||||
@@ -207,7 +231,11 @@ def walk(rel: PurePosixPath, stat: stat_result | None = None) -> list[FileEntry]
|
|||||||
li.append((int(isfile), f.name, s))
|
li.append((int(isfile), f.name, s))
|
||||||
# Build the tree as a list of FileEntries
|
# Build the tree as a list of FileEntries
|
||||||
for [_, name, s] in humansorted(li):
|
for [_, name, s] in humansorted(li):
|
||||||
|
if stop_event.is_set():
|
||||||
|
raise _WatcherStoppingError
|
||||||
sub = walk(rel / name, stat=s)
|
sub = walk(rel / name, stat=s)
|
||||||
|
if not sub:
|
||||||
|
continue
|
||||||
child = sub[0]
|
child = sub[0]
|
||||||
entry = FileEntry(
|
entry = FileEntry(
|
||||||
level=entry.level,
|
level=entry.level,
|
||||||
@@ -244,6 +272,7 @@ def update_root(loop):
|
|||||||
def update_path(rootmod: list[FileEntry], relpath: PurePosixPath, loop):
|
def update_path(rootmod: list[FileEntry], relpath: PurePosixPath, loop):
|
||||||
"""Called on FS updates, check the filesystem and broadcast any changes."""
|
"""Called on FS updates, check the filesystem and broadcast any changes."""
|
||||||
new = walk(relpath)
|
new = walk(relpath)
|
||||||
|
_ = loop
|
||||||
obegin, old = treeget(rootmod, relpath)
|
obegin, old = treeget(rootmod, relpath)
|
||||||
|
|
||||||
if old == new:
|
if old == new:
|
||||||
@@ -300,7 +329,7 @@ def format_update(old, new):
|
|||||||
logger.error(
|
logger.error(
|
||||||
f"format_update potential infinite loop! iteration={iteration_count}, oidx={oidx}, nidx={nidx}"
|
f"format_update potential infinite loop! iteration={iteration_count}, oidx={oidx}, nidx={nidx}"
|
||||||
)
|
)
|
||||||
raise Exception(
|
raise FormatUpdateLoopError(
|
||||||
f"format_update infinite loop detected at iteration {iteration_count}"
|
f"format_update infinite loop detected at iteration {iteration_count}"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -641,8 +670,6 @@ def watcher(loop):
|
|||||||
modified_flags = frozenset()
|
modified_flags = frozenset()
|
||||||
|
|
||||||
if use_inotify:
|
if use_inotify:
|
||||||
import inotify.adapters
|
|
||||||
|
|
||||||
modified_flags = frozenset(
|
modified_flags = frozenset(
|
||||||
(
|
(
|
||||||
"IN_CREATE",
|
"IN_CREATE",
|
||||||
@@ -657,12 +684,13 @@ def watcher(loop):
|
|||||||
|
|
||||||
while not stop_event.is_set():
|
while not stop_event.is_set():
|
||||||
if use_inotify:
|
if use_inotify:
|
||||||
import inotify.adapters
|
|
||||||
|
|
||||||
inotify_tree = inotify.adapters.InotifyTree(rootpath.as_posix())
|
inotify_tree = inotify.adapters.InotifyTree(rootpath.as_posix())
|
||||||
|
|
||||||
# Initialize the tree from filesystem
|
# Initialize the tree from filesystem
|
||||||
|
try:
|
||||||
update_root(loop)
|
update_root(loop)
|
||||||
|
except _WatcherStoppingError:
|
||||||
|
return
|
||||||
path_index = PathIndex(state.root[:])
|
path_index = PathIndex(state.root[:])
|
||||||
|
|
||||||
trefresh = time.monotonic() + 300.0
|
trefresh = time.monotonic() + 300.0
|
||||||
@@ -746,7 +774,10 @@ def watcher(loop):
|
|||||||
# Process each collapsed path
|
# Process each collapsed path
|
||||||
new_root = path_index.root
|
new_root = path_index.root
|
||||||
for path in collapsed:
|
for path in collapsed:
|
||||||
|
try:
|
||||||
new_entries = walk(path)
|
new_entries = walk(path)
|
||||||
|
except _WatcherStoppingError:
|
||||||
|
return
|
||||||
new_root = path_index.apply_update(path, new_entries)
|
new_root = path_index.apply_update(path, new_entries)
|
||||||
|
|
||||||
# Broadcast if changed
|
# Broadcast if changed
|
||||||
@@ -765,12 +796,51 @@ def watcher(loop):
|
|||||||
with state.lock:
|
with state.lock:
|
||||||
broadcast(update_msg, loop)
|
broadcast(update_msg, loop)
|
||||||
state.root = fresh
|
state.root = fresh
|
||||||
|
except _WatcherStoppingError:
|
||||||
|
return
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Fallback failed; sending full root")
|
logger.exception("Fallback failed; sending full root")
|
||||||
with state.lock:
|
with state.lock:
|
||||||
broadcast(format_root(fresh), loop)
|
broadcast(format_root(fresh), loop)
|
||||||
state.root = fresh
|
state.root = fresh
|
||||||
|
|
||||||
|
# Drain AR updates from preview worker (immediate, no debounce)
|
||||||
|
ar_new_root: list[FileEntry] | None = None
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
fuid_key, ar = _ar_queue.get_nowait()
|
||||||
|
_ar_map[fuid_key] = ar
|
||||||
|
# Patch the matching entry in the current root
|
||||||
|
root_to_patch = (
|
||||||
|
ar_new_root if ar_new_root is not None else path_index.root
|
||||||
|
)
|
||||||
|
for i, entry in enumerate(root_to_patch):
|
||||||
|
if entry.key == fuid_key and entry.isfile and entry.ar != ar:
|
||||||
|
if ar_new_root is None:
|
||||||
|
ar_new_root = root_to_patch[:]
|
||||||
|
ar_new_root[i] = FileEntry(
|
||||||
|
level=entry.level,
|
||||||
|
name=entry.name,
|
||||||
|
key=entry.key,
|
||||||
|
mtime=entry.mtime,
|
||||||
|
size=entry.size,
|
||||||
|
allocated=entry.allocated,
|
||||||
|
isfile=entry.isfile,
|
||||||
|
ar=ar,
|
||||||
|
)
|
||||||
|
break
|
||||||
|
except queue.Empty:
|
||||||
|
pass
|
||||||
|
if ar_new_root is not None:
|
||||||
|
try:
|
||||||
|
update_msg = format_update(state.root, ar_new_root)
|
||||||
|
with state.lock:
|
||||||
|
broadcast(update_msg, loop)
|
||||||
|
state.root = ar_new_root
|
||||||
|
path_index = PathIndex(ar_new_root)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("AR update broadcast failed")
|
||||||
|
|
||||||
# Collect events from websocket signals (non-blocking)
|
# Collect events from websocket signals (non-blocking)
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
@@ -816,6 +886,7 @@ def start(app):
|
|||||||
global rootpath
|
global rootpath
|
||||||
config.load_config()
|
config.load_config()
|
||||||
rootpath = config.config.path
|
rootpath = config.config.path
|
||||||
|
stop_event.clear()
|
||||||
app.ctx.watcher = threading.Thread(
|
app.ctx.watcher = threading.Thread(
|
||||||
target=watcher,
|
target=watcher,
|
||||||
args=[app.loop],
|
args=[app.loop],
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
services:
|
||||||
|
onlyoffice:
|
||||||
|
build:
|
||||||
|
context: ./docker/onlyoffice-converter-patch
|
||||||
|
args:
|
||||||
|
ONLYOFFICE_VERSION: "9.3.1"
|
||||||
|
container_name: onlyoffice
|
||||||
|
ports:
|
||||||
|
- "8080:80"
|
||||||
|
environment:
|
||||||
|
# Number of converter workers (default 8).
|
||||||
|
# Set to your CPU count or slightly below.
|
||||||
|
- WORKERS
|
||||||
|
# JWT secret shared with Cista.
|
||||||
|
# OnlyOffice reads it as JWT_SECRET; Cista reads it as ONLYOFFICE_JWT_SECRET.
|
||||||
|
# We use ONLYOFFICE_JWT_SECRET as the canonical name so you only set one variable.
|
||||||
|
- JWT_SECRET=${ONLYOFFICE_JWT_SECRET}
|
||||||
|
- JWT_ENABLED=true
|
||||||
|
- JWT_HEADER=Authorization
|
||||||
|
volumes:
|
||||||
|
# Persist fonts and generated caches across restarts
|
||||||
|
- onlyoffice-data:/var/www/onlyoffice/Data
|
||||||
|
- onlyoffice-lib:/var/lib/onlyoffice
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
onlyoffice-data:
|
||||||
|
onlyoffice-lib:
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Patched OnlyOffice Document Server with configurable converter worker count.
|
||||||
|
#
|
||||||
|
# The Community Edition hardcodes the document converter to 1 worker,
|
||||||
|
# which creates a severe bottleneck under concurrent load.
|
||||||
|
# This image patches the open-source license.js to spawn a configurable
|
||||||
|
# number of converter workers (default 8).
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build -t onlyoffice-cista docker/onlyoffice-converter-patch
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker run -d -p 8988:80 \
|
||||||
|
# -e WORKERS=16 \
|
||||||
|
# -e JWT_SECRET=your-strong-secret \
|
||||||
|
# --name onlyoffice onlyoffice-cista
|
||||||
|
#
|
||||||
|
# JWT:
|
||||||
|
# Set JWT_SECRET to the same value you pass to Cista as ONLYOFFICE_JWT_SECRET.
|
||||||
|
# OnlyOffice will enable token validation automatically.
|
||||||
|
#
|
||||||
|
# The ONLYOFFICE_VERSION build arg lets you target a specific release.
|
||||||
|
|
||||||
|
ARG ONLYOFFICE_VERSION=9.3.1
|
||||||
|
|
||||||
|
FROM onlyoffice/documentserver:${ONLYOFFICE_VERSION}
|
||||||
|
|
||||||
|
# Prevent interactive apt prompts
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# Install Node.js, npm, and git so we can run the FileConverter from source.
|
||||||
|
RUN apt-get update -qq && \
|
||||||
|
apt-get install -y -qq --no-install-recommends \
|
||||||
|
nodejs \
|
||||||
|
npm \
|
||||||
|
git \
|
||||||
|
ca-certificates && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Clone the open-source server components (shallow, ~15 MB).
|
||||||
|
# The master branch is used because the Linux/web tags are not published
|
||||||
|
# in the server repo; the license.js file has been stable for years.
|
||||||
|
RUN git clone --depth 1 https://github.com/ONLYOFFICE/server.git /opt/oo-server
|
||||||
|
|
||||||
|
# Patch license.js so the converter worker count is read from an env var
|
||||||
|
# instead of being hardcoded to 1.
|
||||||
|
RUN sed -i \
|
||||||
|
's/count: 1,/count: parseInt(process.env.WORKERS, 10) || 8,/' \
|
||||||
|
/opt/oo-server/Common/sources/license.js
|
||||||
|
|
||||||
|
# Install npm dependencies for the modules the FileConverter touches.
|
||||||
|
# DocService deps are also needed because converter.js pulls in baseConnector.
|
||||||
|
RUN cd /opt/oo-server/Common && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/FileConverter && npm ci --no-audit --no-fund
|
||||||
|
RUN cd /opt/oo-server/DocService && npm ci --no-audit --no-fund
|
||||||
|
|
||||||
|
# Back up the compiled pkg binary and replace it with our wrapper.
|
||||||
|
RUN mv /var/www/onlyoffice/documentserver/server/FileConverter/converter \
|
||||||
|
/var/www/onlyoffice/documentserver/server/FileConverter/converter.orig
|
||||||
|
|
||||||
|
COPY converter-wrapper.sh /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
RUN chmod +x /var/www/onlyoffice/documentserver/server/FileConverter/converter
|
||||||
|
|
||||||
|
# Default worker count (override at runtime with -e WORKERS=16).
|
||||||
|
ENV WORKERS=8
|
||||||
|
|
||||||
|
# Use our custom entrypoint to persist the env var to a file that the
|
||||||
|
# non-root converter process (user=ds) can read.
|
||||||
|
COPY entrypoint.sh /app/ds/run-document-server-patched.sh
|
||||||
|
RUN chmod +x /app/ds/run-document-server-patched.sh
|
||||||
|
ENTRYPOINT ["/app/ds/run-document-server-patched.sh"]
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Wrapper that runs the OnlyOffice FileConverter from patched Node.js source.
|
||||||
|
# Replaces the compiled pkg binary shipped with the Community Edition.
|
||||||
|
|
||||||
|
# The env var is not passed through supervisor to the 'ds' user, so we read
|
||||||
|
# it from a file written by the custom entrypoint.
|
||||||
|
if [ -z "${WORKERS}" ] && [ -r /tmp/oo-converter-workers.txt ]; then
|
||||||
|
export WORKERS=$(cat /tmp/oo-converter-workers.txt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd /opt/oo-server/FileConverter || exit 1
|
||||||
|
|
||||||
|
export NODE_ENV=production-linux
|
||||||
|
export NODE_CONFIG_DIR=/etc/onlyoffice/documentserver
|
||||||
|
export NODE_DISABLE_COLORS=1
|
||||||
|
export APPLICATION_NAME=onlyoffice
|
||||||
|
export LD_LIBRARY_PATH=/var/www/onlyoffice/documentserver/server/FileConverter/bin
|
||||||
|
|
||||||
|
exec node sources/convertermaster.js "$@"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Custom entrypoint that persists WORKERS to a file readable by
|
||||||
|
# the non-root user that supervisor uses to run the converter.
|
||||||
|
|
||||||
|
echo "${WORKERS:-8}" > /tmp/oo-converter-workers.txt
|
||||||
|
chmod 644 /tmp/oo-converter-workers.txt
|
||||||
|
|
||||||
|
exec /app/ds/run-document-server.sh "$@"
|
||||||
@@ -3,7 +3,13 @@
|
|||||||
<GalleryFigure v-if="editing?.key === 'new'" :doc="editing" :key=editing.key :editing="{rename: mkdir, exit}" />
|
<GalleryFigure v-if="editing?.key === 'new'" :doc="editing" :key=editing.key :editing="{rename: mkdir, exit}" />
|
||||||
<template v-for="(doc, index) in documents" :key=doc.key>
|
<template v-for="(doc, index) in documents" :key=doc.key>
|
||||||
<BreadCrumb v-if="showFolderBreadcrumb(index)" :path="doc.loc ? doc.loc.split('/') : []" class="folder-indicator"/>
|
<BreadCrumb v-if="showFolderBreadcrumb(index)" :path="doc.loc ? doc.loc.split('/') : []" class="folder-indicator"/>
|
||||||
<GalleryFigure :doc=doc :editing="editing === doc ? {rename, exit} : null" @menu="contextMenu($event, doc)" :class="{ 'folder-start': showFolderBreadcrumb(index) }" />
|
<GalleryFigure
|
||||||
|
:doc=doc
|
||||||
|
:editing="editing === doc ? {rename, exit} : null"
|
||||||
|
:style="{ '--gallery-figure-height': rowHeightsByKey[doc.key] ?? '15em' }"
|
||||||
|
@menu="contextMenu($event, doc)"
|
||||||
|
:class="{ 'folder-start': showFolderBreadcrumb(index) }"
|
||||||
|
/>
|
||||||
</template>
|
</template>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -21,6 +27,7 @@ import {
|
|||||||
onUnmounted,
|
onUnmounted,
|
||||||
ref,
|
ref,
|
||||||
shallowRef,
|
shallowRef,
|
||||||
|
watch,
|
||||||
watchEffect
|
watchEffect
|
||||||
} from 'vue'
|
} from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter } from 'vue-router'
|
||||||
@@ -71,11 +78,108 @@ const rename = async (doc: Doc, newName: string) => {
|
|||||||
}
|
}
|
||||||
const gallery = ref<HTMLElement>()
|
const gallery = ref<HTMLElement>()
|
||||||
const columnCount = ref(1)
|
const columnCount = ref(1)
|
||||||
|
const columnWidthPx = ref(240)
|
||||||
|
const emPx = ref(16)
|
||||||
|
const aspectByKey = ref<Record<string, number>>({})
|
||||||
|
|
||||||
|
const optimalRowHeightPx = (ratios: number[]) => {
|
||||||
|
const w = Math.max(1, columnWidthPx.value)
|
||||||
|
const minH = Math.max(1, Math.round(7 * emPx.value))
|
||||||
|
const maxH = Math.max(minH, Math.round(25 * emPx.value))
|
||||||
|
const usable = ratios.filter(ar => Number.isFinite(ar) && ar > 0)
|
||||||
|
if (usable.length === 0) return Math.round(15 * emPx.value)
|
||||||
|
|
||||||
|
let bestH = Math.round(15 * emPx.value)
|
||||||
|
let bestScore = -1
|
||||||
|
for (let h = minH; h <= maxH; h++) {
|
||||||
|
let score = 0
|
||||||
|
for (const ar of usable) {
|
||||||
|
let shownW = w
|
||||||
|
let shownH = w * ar
|
||||||
|
if (shownH > h) {
|
||||||
|
shownH = h
|
||||||
|
shownW = h / ar
|
||||||
|
}
|
||||||
|
// Fill efficiency in the row cell (0..1)
|
||||||
|
score += (shownW * shownH) / (w * h)
|
||||||
|
}
|
||||||
|
if (score > bestScore) {
|
||||||
|
bestScore = score
|
||||||
|
bestH = h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bestH
|
||||||
|
}
|
||||||
|
|
||||||
|
const setAspect = (key: string, ar: number) => {
|
||||||
|
if (!Number.isFinite(ar) || ar <= 0) return
|
||||||
|
if (aspectByKey.value[key] === ar) return
|
||||||
|
aspectByKey.value = {
|
||||||
|
...aspectByKey.value,
|
||||||
|
[key]: ar
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const rowHeightsByKey = computed<Record<string, string>>(() => {
|
||||||
|
const docs = props.documents
|
||||||
|
const cols = Math.max(1, columnCount.value)
|
||||||
|
const byKey = aspectByKey.value
|
||||||
|
const out: Record<string, string> = {}
|
||||||
|
|
||||||
|
const assignRows = (group: Doc[]) => {
|
||||||
|
for (let start = 0; start < group.length; start += cols) {
|
||||||
|
const row = group.slice(start, start + cols)
|
||||||
|
const ratios = row
|
||||||
|
.filter(doc => doc.previewable)
|
||||||
|
.map(doc => byKey[doc.key])
|
||||||
|
.filter((ar): ar is number => ar != null)
|
||||||
|
const height = `${optimalRowHeightPx(ratios)}px`
|
||||||
|
for (const doc of row) out[doc.key] = height
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let group: Doc[] = []
|
||||||
|
for (let i = 0; i < docs.length; i++) {
|
||||||
|
if (i > 0 && docs[i]!.loc !== docs[i - 1]!.loc) {
|
||||||
|
assignRows(group)
|
||||||
|
group = []
|
||||||
|
}
|
||||||
|
group.push(docs[i]!)
|
||||||
|
}
|
||||||
|
assignRows(group)
|
||||||
|
|
||||||
|
return out
|
||||||
|
})
|
||||||
|
|
||||||
|
// Seed collected ratios from server-provided ar values on docs
|
||||||
|
const seedFromDocs = () => {
|
||||||
|
for (const doc of props.documents)
|
||||||
|
if (doc.previewable && doc.ar != null) setAspect(doc.key, doc.ar)
|
||||||
|
}
|
||||||
|
|
||||||
|
const onImgLoad = (e: Event) => {
|
||||||
|
const img = e.target as HTMLImageElement
|
||||||
|
if (img.tagName !== 'IMG' || img.naturalWidth === 0) return
|
||||||
|
const anchor = img.closest('a[id^="file-"]') as HTMLAnchorElement | null
|
||||||
|
if (!anchor) return
|
||||||
|
const key = anchor.id.slice('file-'.length)
|
||||||
|
if (!key) return
|
||||||
|
setAspect(key, img.naturalHeight / img.naturalWidth)
|
||||||
|
}
|
||||||
const updateColumns = () => {
|
const updateColumns = () => {
|
||||||
if (!gallery.value) return
|
if (!gallery.value) return
|
||||||
columnCount.value = getComputedStyle(gallery.value).gridTemplateColumns.split(
|
const style = getComputedStyle(gallery.value)
|
||||||
' '
|
const templates = style.gridTemplateColumns
|
||||||
).length
|
.split(' ')
|
||||||
|
.filter(part => !!part && part !== 'none')
|
||||||
|
columnCount.value = Math.max(1, templates.length)
|
||||||
|
const first = templates[0]
|
||||||
|
if (first && first.endsWith('px')) {
|
||||||
|
const parsed = Number.parseFloat(first)
|
||||||
|
if (Number.isFinite(parsed) && parsed > 0) columnWidthPx.value = parsed
|
||||||
|
}
|
||||||
|
const parsedEm = Number.parseFloat(style.fontSize)
|
||||||
|
if (Number.isFinite(parsedEm) && parsedEm > 0) emPx.value = parsedEm
|
||||||
}
|
}
|
||||||
const columns = computed(() => columnCount.value)
|
const columns = computed(() => columnCount.value)
|
||||||
defineExpose({
|
defineExpose({
|
||||||
@@ -230,14 +334,20 @@ onMounted(() => {
|
|||||||
active.focus()
|
active.focus()
|
||||||
}
|
}
|
||||||
updateColumns()
|
updateColumns()
|
||||||
|
seedFromDocs()
|
||||||
if (gallery.value) {
|
if (gallery.value) {
|
||||||
resizeObserver = new ResizeObserver(updateColumns)
|
resizeObserver = new ResizeObserver(updateColumns)
|
||||||
resizeObserver.observe(gallery.value)
|
resizeObserver.observe(gallery.value)
|
||||||
|
gallery.value.addEventListener('load', onImgLoad, { capture: true })
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
onUnmounted(() => {
|
onUnmounted(() => {
|
||||||
resizeObserver?.disconnect()
|
resizeObserver?.disconnect()
|
||||||
|
gallery.value?.removeEventListener('load', onImgLoad, { capture: true })
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Re-seed aspect ratios whenever docs update (e.g., ar patch from server)
|
||||||
|
watch(() => props.documents, seedFromDocs)
|
||||||
const mkdir = async (doc: Doc, name: string) => {
|
const mkdir = async (doc: Doc, name: string) => {
|
||||||
doc.name = name
|
doc.name = name
|
||||||
doc.key = crypto.randomUUID()
|
doc.key = crypto.randomUUID()
|
||||||
|
|||||||
@@ -82,7 +82,8 @@ const onclick = (ev: Event) => {
|
|||||||
margin-left: 0.3em;
|
margin-left: 0.3em;
|
||||||
}
|
}
|
||||||
figure {
|
figure {
|
||||||
max-height: 15em;
|
height: var(--gallery-figure-height, 15em);
|
||||||
|
max-height: var(--gallery-figure-height, 15em);
|
||||||
position: relative;
|
position: relative;
|
||||||
border-radius: .5em;
|
border-radius: .5em;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -92,12 +93,13 @@ figure {
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
transition: height 0.4s ease, max-height 0.4s ease;
|
||||||
}
|
}
|
||||||
figure > article {
|
figure > article {
|
||||||
flex: 0 0 auto;
|
flex: 0 0 auto;
|
||||||
}
|
}
|
||||||
figure :deep(.video-container) {
|
figure :deep(.video-container) {
|
||||||
height: 15em;
|
height: var(--gallery-figure-height, 15em);
|
||||||
}
|
}
|
||||||
.titlespacer {
|
.titlespacer {
|
||||||
flex-shrink: 100000;
|
flex-shrink: 100000;
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { Play as PlayIcon, Spinner as SpinnerIcon } from '@/assets/svg'
|
import { Play as PlayIcon, Spinner as SpinnerIcon } from '@/assets/svg'
|
||||||
import type { Doc } from '@/repositories/Document'
|
import type { Doc } from '@/repositories/Document'
|
||||||
|
import { useMainStore } from '@/stores/main'
|
||||||
import { computed, ref } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
|
|
||||||
const aud = ref<HTMLAudioElement | null>(null)
|
const aud = ref<HTMLAudioElement | null>(null)
|
||||||
@@ -125,64 +126,22 @@ defineExpose({
|
|||||||
media
|
media
|
||||||
})
|
})
|
||||||
|
|
||||||
const video = () => ['mkv', 'mp4', 'webm', 'mov', 'avi'].includes(props.doc.ext)
|
const video = () => props.doc.video
|
||||||
const audio = () => ['mp3', 'flac', 'ogg', 'aac'].includes(props.doc.ext)
|
const audio = () => props.doc.audio
|
||||||
const archive = () =>
|
const archive = () => props.doc.archive
|
||||||
['zip', 'tar', 'gz', 'bz2', 'xz', '7z', 'rar'].includes(props.doc.ext)
|
const docs = () => props.doc.document
|
||||||
|
// image = requires server-side preview (browsers cannot display it natively)
|
||||||
|
// img = browser-viewable image that can be used directly in an <img> tag
|
||||||
|
const image = () => props.doc.image
|
||||||
|
const print = () => props.doc.print
|
||||||
const showProgress = () => !props.doc.complete && (preview() || props.doc.img)
|
const showProgress = () => !props.doc.complete && (preview() || props.doc.img)
|
||||||
const preview = () =>
|
const preview = () => {
|
||||||
[
|
const store = useMainStore()
|
||||||
'bmp',
|
return (
|
||||||
'ico',
|
!(store.server.office_previews === false && docs()) &&
|
||||||
'tif',
|
(image() || print() || (props.doc.img && props.doc.size > 500000))
|
||||||
'tiff',
|
)
|
||||||
'heic',
|
}
|
||||||
'heif',
|
|
||||||
'pdf',
|
|
||||||
'epub',
|
|
||||||
'mobi',
|
|
||||||
// Documents
|
|
||||||
'doc',
|
|
||||||
'dot',
|
|
||||||
'docx',
|
|
||||||
'docm',
|
|
||||||
'dotx',
|
|
||||||
'dotm',
|
|
||||||
'rtf',
|
|
||||||
'odt',
|
|
||||||
'ott',
|
|
||||||
'txt',
|
|
||||||
'md',
|
|
||||||
'mhtml',
|
|
||||||
'mht',
|
|
||||||
'html',
|
|
||||||
'htm',
|
|
||||||
'xml',
|
|
||||||
'wps',
|
|
||||||
'wri',
|
|
||||||
// Spreadsheets
|
|
||||||
'xls',
|
|
||||||
'xlsx',
|
|
||||||
'xlsm',
|
|
||||||
'xlsb',
|
|
||||||
'xltx',
|
|
||||||
'xltm',
|
|
||||||
'ods',
|
|
||||||
'ots',
|
|
||||||
'csv',
|
|
||||||
// Presentations
|
|
||||||
'ppt',
|
|
||||||
'pptx',
|
|
||||||
'pptm',
|
|
||||||
'pps',
|
|
||||||
'ppsx',
|
|
||||||
'pot',
|
|
||||||
'potx',
|
|
||||||
'odp',
|
|
||||||
'otp'
|
|
||||||
].includes(props.doc.ext) ||
|
|
||||||
(props.doc.size > 500000 &&
|
|
||||||
['avif', 'webp', 'png', 'jpg', 'jpeg'].includes(props.doc.ext))
|
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { formatSize, formatUnixDate } from '@/utils'
|
import { useMainStore } from '@/stores/main'
|
||||||
|
import { FILE_TYPES, formatSize, formatUnixDate } from '@/utils'
|
||||||
|
|
||||||
export type FUID = string
|
export type FUID = string
|
||||||
|
|
||||||
@@ -12,6 +13,7 @@ export type DocProps = {
|
|||||||
dir: boolean
|
dir: boolean
|
||||||
ghost?: boolean
|
ghost?: boolean
|
||||||
expires?: number // Unix timestamp for ghost expiry
|
expires?: number // Unix timestamp for ghost expiry
|
||||||
|
ar?: number // Aspect ratio (height/width) from server, if known
|
||||||
}
|
}
|
||||||
|
|
||||||
export class Doc {
|
export class Doc {
|
||||||
@@ -25,6 +27,7 @@ export class Doc {
|
|||||||
public expires: number = 0 // Unix timestamp for ghost expiry (0 = no expiry)
|
public expires: number = 0 // Unix timestamp for ghost expiry (0 = no expiry)
|
||||||
/** @internal Use the name getter/setter instead */
|
/** @internal Use the name getter/setter instead */
|
||||||
public _name: string = ''
|
public _name: string = ''
|
||||||
|
public ar?: number // Aspect ratio (height/width), provided by server after first preview render
|
||||||
|
|
||||||
constructor(props: Partial<DocProps> = {}) {
|
constructor(props: Partial<DocProps> = {}) {
|
||||||
const { name, ...rest } = props
|
const { name, ...rest } = props
|
||||||
@@ -63,86 +66,53 @@ export class Doc {
|
|||||||
return this.url.replace(/^\/#/, '')
|
return this.url.replace(/^\/#/, '')
|
||||||
}
|
}
|
||||||
get img(): boolean {
|
get img(): boolean {
|
||||||
// Folders cannot be images
|
return (
|
||||||
if (this.dir) return false
|
!this.dir && (FILE_TYPES.imageBrowser as readonly string[]).includes(this.ext)
|
||||||
return [
|
)
|
||||||
'jpg',
|
}
|
||||||
'jpeg',
|
get video(): boolean {
|
||||||
'png',
|
return (FILE_TYPES.video as readonly string[]).includes(this.ext)
|
||||||
'gif',
|
}
|
||||||
'webp',
|
get audio(): boolean {
|
||||||
'avif',
|
return (FILE_TYPES.audio as readonly string[]).includes(this.ext)
|
||||||
'heic',
|
}
|
||||||
'heif',
|
get archive(): boolean {
|
||||||
'svg'
|
return (FILE_TYPES.archive as readonly string[]).includes(this.ext)
|
||||||
].includes(this.ext)
|
}
|
||||||
|
get document(): boolean {
|
||||||
|
return (FILE_TYPES.document as readonly string[]).includes(this.ext)
|
||||||
|
}
|
||||||
|
// Images that require server-side preview (browsers cannot display them natively)
|
||||||
|
get image(): boolean {
|
||||||
|
return (FILE_TYPES.image as readonly string[]).includes(this.ext)
|
||||||
|
}
|
||||||
|
get print(): boolean {
|
||||||
|
return (FILE_TYPES.print as readonly string[]).includes(this.ext)
|
||||||
}
|
}
|
||||||
get complete(): boolean {
|
get complete(): boolean {
|
||||||
return !this.ghost && (this.dir || this.size <= this.allocated)
|
return !this.ghost && (this.dir || this.size <= this.allocated)
|
||||||
}
|
}
|
||||||
get previewable(): boolean {
|
get previewable(): boolean {
|
||||||
// Folders cannot be previewable
|
|
||||||
if (this.dir) return false
|
if (this.dir) return false
|
||||||
if (this.img) return true
|
return (
|
||||||
// Not a comprehensive list, but good enough for now
|
this.img ||
|
||||||
return [
|
this.video ||
|
||||||
'mp4',
|
this.audio ||
|
||||||
'mkv',
|
this.image ||
|
||||||
'webm',
|
this.print ||
|
||||||
'ogg',
|
(this.document && useMainStore().server.office_previews !== false)
|
||||||
'mp3',
|
)
|
||||||
'flac',
|
|
||||||
'aac',
|
|
||||||
'pdf',
|
|
||||||
// Documents
|
|
||||||
'doc',
|
|
||||||
'dot',
|
|
||||||
'docx',
|
|
||||||
'docm',
|
|
||||||
'dotx',
|
|
||||||
'dotm',
|
|
||||||
'rtf',
|
|
||||||
'odt',
|
|
||||||
'ott',
|
|
||||||
'txt',
|
|
||||||
'md',
|
|
||||||
'mhtml',
|
|
||||||
'mht',
|
|
||||||
'html',
|
|
||||||
'htm',
|
|
||||||
'xml',
|
|
||||||
'wps',
|
|
||||||
'wri',
|
|
||||||
// Spreadsheets
|
|
||||||
'xls',
|
|
||||||
'xlsx',
|
|
||||||
'xlsm',
|
|
||||||
'xlsb',
|
|
||||||
'xltx',
|
|
||||||
'xltm',
|
|
||||||
'ods',
|
|
||||||
'ots',
|
|
||||||
'csv',
|
|
||||||
// Presentations
|
|
||||||
'ppt',
|
|
||||||
'pptx',
|
|
||||||
'pptm',
|
|
||||||
'pps',
|
|
||||||
'ppsx',
|
|
||||||
'pot',
|
|
||||||
'potx',
|
|
||||||
'odp',
|
|
||||||
'otp'
|
|
||||||
].includes(this.ext)
|
|
||||||
}
|
}
|
||||||
get previewurl(): string {
|
get previewurl(): string {
|
||||||
if (!this.complete || !this.previewable) return ''
|
return !this.complete || !this.previewable
|
||||||
return this.url.replace(/^\/files/, '/preview')
|
? ''
|
||||||
|
: this.url.replace(/^\/files/, '/preview')
|
||||||
}
|
}
|
||||||
get ext(): string {
|
get ext(): string {
|
||||||
const dotIndex = this.name.lastIndexOf('.')
|
const dotIndex = this.name.lastIndexOf('.')
|
||||||
if (dotIndex === -1 || dotIndex === this.name.length - 1) return ''
|
return dotIndex === -1 || dotIndex === this.name.length - 1
|
||||||
return this.name.slice(dotIndex + 1).toLowerCase()
|
? ''
|
||||||
|
: this.name.slice(dotIndex + 1).toLowerCase()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
export type errorEvent = {
|
export type errorEvent = {
|
||||||
@@ -162,7 +132,8 @@ export type FileEntry = [
|
|||||||
number, // mtime
|
number, // mtime
|
||||||
number, // size
|
number, // size
|
||||||
number, // allocated (actual disk usage)
|
number, // allocated (actual disk usage)
|
||||||
number // isfile
|
number, // isfile
|
||||||
|
number? // ar: aspect ratio (height/width), present if known
|
||||||
]
|
]
|
||||||
|
|
||||||
export type UpdateEntry = ['k', number] | ['d', number] | ['i', Array<FileEntry>]
|
export type UpdateEntry = ['k', number] | ['d', number] | ['i', Array<FileEntry>]
|
||||||
|
|||||||
@@ -164,6 +164,11 @@ const handleWatchMessage = (event: MessageEvent) => {
|
|||||||
case !!msg.update:
|
case !!msg.update:
|
||||||
handleUpdateMessage(msg)
|
handleUpdateMessage(msg)
|
||||||
break
|
break
|
||||||
|
case !!msg.ar: {
|
||||||
|
const store = useMainStore()
|
||||||
|
store.updateAr(msg.ar as Record<string, number>)
|
||||||
|
break
|
||||||
|
}
|
||||||
case !!msg.space:
|
case !!msg.space:
|
||||||
const store = useMainStore()
|
const store = useMainStore()
|
||||||
store.space = msg.space
|
store.space = msg.space
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { collator } from '@/utils'
|
|||||||
import { type SortOrder, sorted } from '@/utils/docsort'
|
import { type SortOrder, sorted } from '@/utils/docsort'
|
||||||
import SearchWorker from '@/workers/searchWorker?worker'
|
import SearchWorker from '@/workers/searchWorker?worker'
|
||||||
import { type StateTree, defineStore } from 'pinia'
|
import { type StateTree, defineStore } from 'pinia'
|
||||||
import { documentRef, getDocuments, setDocuments } from './documentStore'
|
import { documentRef, getDocuments, setDocuments, triggerUpdate } from './documentStore'
|
||||||
|
|
||||||
// Singleton search worker instance
|
// Singleton search worker instance
|
||||||
let searchWorker: Worker | null = null
|
let searchWorker: Worker | null = null
|
||||||
@@ -79,7 +79,11 @@ export const useMainStore = defineStore('main', {
|
|||||||
connected: false,
|
connected: false,
|
||||||
authInProgress: false,
|
authInProgress: false,
|
||||||
cursor: '' as string,
|
cursor: '' as string,
|
||||||
server: {} as Record<string, any> & { public?: boolean; paskia?: boolean },
|
server: {} as Record<string, any> & {
|
||||||
|
public?: boolean
|
||||||
|
paskia?: boolean
|
||||||
|
office_previews?: boolean
|
||||||
|
},
|
||||||
dialog: '' as '' | 'settings' | 'usermgmt' | 'accessdenied' | 'tokens',
|
dialog: '' as '' | 'settings' | 'usermgmt' | 'accessdenied' | 'tokens',
|
||||||
uprogress: {} as any,
|
uprogress: {} as any,
|
||||||
dprogress: {} as any,
|
dprogress: {} as any,
|
||||||
@@ -120,7 +124,7 @@ export const useMainStore = defineStore('main', {
|
|||||||
updateRoot(root: FileEntry[]) {
|
updateRoot(root: FileEntry[]) {
|
||||||
const docs = []
|
const docs = []
|
||||||
let loc = [] as string[]
|
let loc = [] as string[]
|
||||||
for (const [level, name, key, mtime, size, allocated, isfile] of root) {
|
for (const [level, name, key, mtime, size, allocated, isfile, ar] of root) {
|
||||||
loc = loc.slice(0, level - 1)
|
loc = loc.slice(0, level - 1)
|
||||||
docs.push(
|
docs.push(
|
||||||
new Doc({
|
new Doc({
|
||||||
@@ -130,7 +134,8 @@ export const useMainStore = defineStore('main', {
|
|||||||
size,
|
size,
|
||||||
allocated,
|
allocated,
|
||||||
mtime,
|
mtime,
|
||||||
dir: !isfile
|
dir: !isfile,
|
||||||
|
ar
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
loc.push(name)
|
loc.push(name)
|
||||||
@@ -153,6 +158,22 @@ export const useMainStore = defineStore('main', {
|
|||||||
// Sync documents to search worker
|
// Sync documents to search worker
|
||||||
this.syncSearchWorker()
|
this.syncSearchWorker()
|
||||||
},
|
},
|
||||||
|
/** Patch aspect ratios on existing docs from a server ar update message */
|
||||||
|
updateAr(arMap: Record<string, number>) {
|
||||||
|
const docs = getDocuments()
|
||||||
|
let changed = false
|
||||||
|
for (const doc of docs) {
|
||||||
|
const ar = arMap[doc.key]
|
||||||
|
if (ar != null && doc.ar !== ar) {
|
||||||
|
doc.ar = ar
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (changed) {
|
||||||
|
triggerUpdate()
|
||||||
|
this.docVersion++
|
||||||
|
}
|
||||||
|
},
|
||||||
/** Add a ghost file/folder for optimistic UI updates */
|
/** Add a ghost file/folder for optimistic UI updates */
|
||||||
addGhost(doc: Doc) {
|
addGhost(doc: Doc) {
|
||||||
doc.ghost = true
|
doc.ghost = true
|
||||||
|
|||||||
+18
-12
@@ -69,23 +69,29 @@ export function getFileExtension(filename: string) {
|
|||||||
}
|
}
|
||||||
return filename.slice(dotIndex + 1)
|
return filename.slice(dotIndex + 1)
|
||||||
}
|
}
|
||||||
interface FileTypes {
|
export const FILE_TYPES = {
|
||||||
[key: string]: string[]
|
|
||||||
}
|
|
||||||
|
|
||||||
const filetypes: FileTypes = {
|
|
||||||
video: ['avi', 'mkv', 'mov', 'mp4', 'webm'],
|
video: ['avi', 'mkv', 'mov', 'mp4', 'webm'],
|
||||||
image: ['avif', 'gif', 'jpg', 'jpeg', 'png', 'webp', 'svg'],
|
audio: ['mp3', 'flac', 'ogg', 'aac'],
|
||||||
pdf: ['pdf']
|
archive: ['zip', 'tar', 'gz', 'bz2', 'xz', '7z', 'rar'],
|
||||||
}
|
document: ['doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'odt', 'ods', 'odp', 'rtf'],
|
||||||
|
imageBrowser: ['avif', 'gif', 'jpg', 'jpeg', 'png', 'webp', 'svg'],
|
||||||
|
// Images that require server-side preview (browsers cannot display them natively)
|
||||||
|
image: ['bmp', 'heic', 'heif', 'ico', 'tif', 'tiff'],
|
||||||
|
print: ['epub', 'mobi', 'pdf']
|
||||||
|
} as const
|
||||||
|
|
||||||
export function getFileType(name: string): string {
|
export type FileCategory = keyof typeof FILE_TYPES
|
||||||
|
|
||||||
|
export function getFileType(name: string): FileCategory | 'unknown' {
|
||||||
const dotIndex = name.lastIndexOf('.')
|
const dotIndex = name.lastIndexOf('.')
|
||||||
if (dotIndex === -1 || dotIndex === name.length - 1) return 'unknown'
|
if (dotIndex === -1 || dotIndex === name.length - 1) return 'unknown'
|
||||||
const ext = name.slice(dotIndex + 1).toLowerCase()
|
const ext = name.slice(dotIndex + 1).toLowerCase()
|
||||||
return (
|
for (const category of Object.keys(FILE_TYPES) as FileCategory[]) {
|
||||||
Object.keys(filetypes).find(type => filetypes[type]!.includes(ext)) || 'unknown'
|
if ((FILE_TYPES[category] as readonly string[]).includes(ext)) {
|
||||||
)
|
return category
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 'unknown'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prebuilt for fast & consistent sorting
|
// Prebuilt for fast & consistent sorting
|
||||||
|
|||||||
+4
-11
@@ -77,8 +77,8 @@ docs = [
|
|||||||
source = "vcs"
|
source = "vcs"
|
||||||
|
|
||||||
[tool.hatch.build]
|
[tool.hatch.build]
|
||||||
artifacts = ["cista/frontend-build"]
|
artifacts = ["cista/frontend-build", "cista/docker"]
|
||||||
targets.sdist.hooks.custom.path = "scripts/fastapi-vue/build-frontend.py"
|
targets.sdist.hooks.custom.path = "scripts/fastapi-vue/buildhook.py"
|
||||||
targets.sdist.include = [
|
targets.sdist.include = [
|
||||||
"/cista",
|
"/cista",
|
||||||
]
|
]
|
||||||
@@ -130,7 +130,6 @@ ignore = [
|
|||||||
"ANN202", # legacy codebase: no full runtime annotation coverage yet
|
"ANN202", # legacy codebase: no full runtime annotation coverage yet
|
||||||
"ANN204", # legacy codebase: no full runtime annotation coverage yet
|
"ANN204", # legacy codebase: no full runtime annotation coverage yet
|
||||||
"ANN205", # legacy codebase: no full runtime annotation coverage yet
|
"ANN205", # legacy codebase: no full runtime annotation coverage yet
|
||||||
"ARG001", # framework and callback signatures commonly require unused args
|
|
||||||
"BLE001", # broad catch remains in boundary/proxy/error-handling paths
|
"BLE001", # broad catch remains in boundary/proxy/error-handling paths
|
||||||
"C901", # legacy complexity; keep other correctness rules enabled
|
"C901", # legacy complexity; keep other correctness rules enabled
|
||||||
"D100", # legacy docs not yet standardized
|
"D100", # legacy docs not yet standardized
|
||||||
@@ -152,22 +151,16 @@ ignore = [
|
|||||||
"EM101", # exception-message style; low signal for this project
|
"EM101", # exception-message style; low signal for this project
|
||||||
"EM102", # exception-message style; low signal for this project
|
"EM102", # exception-message style; low signal for this project
|
||||||
"INP001", # scripts folder intentionally lacks package markers
|
"INP001", # scripts folder intentionally lacks package markers
|
||||||
"PLC0415", # lazy imports used to avoid startup/circular import issues
|
|
||||||
"PLR0911", # legacy complexity; keep other correctness rules enabled
|
"PLR0911", # legacy complexity; keep other correctness rules enabled
|
||||||
"PLR0912", # legacy complexity; keep other correctness rules enabled
|
"PLR0912", # legacy complexity; keep other correctness rules enabled
|
||||||
"PLR0913", # legacy complexity; keep other correctness rules enabled
|
"PLR0913", # legacy complexity; keep other correctness rules enabled
|
||||||
"PLR0915", # legacy complexity; keep other correctness rules enabled
|
"PLR0915", # legacy complexity; keep other correctness rules enabled
|
||||||
"PLR2004", # legacy comparisons use inline constants
|
"PLR2004", # we like magic numbers (don't remove this suppression)
|
||||||
"PLW0603", # module-level shared state exists in server runtime code
|
"PLW0603", # module-level shared state exists in server runtime code
|
||||||
"SLF001", # cohesive modules occasionally need private-member access
|
|
||||||
"TRY002", # exception-class strictness too noisy on legacy handlers
|
|
||||||
"TRY003", # exception-message strictness too noisy on legacy handlers
|
"TRY003", # exception-message strictness too noisy on legacy handlers
|
||||||
"TRY004", # type-check strictness too noisy on legacy handlers
|
|
||||||
"TRY300", # stylistic try/else preference
|
|
||||||
"TRY301", # stylistic raise-in-try preference
|
|
||||||
]
|
]
|
||||||
isort.known-first-party = ["cista"]
|
isort.known-first-party = ["cista"]
|
||||||
per-file-ignores."tests/*" = ["S", "ANN", "D", "INP", "PLR2004"]
|
per-file-ignores."tests/*" = ["S", "ANN", "D", "INP", "PLR2004", "ARG001"]
|
||||||
per-file-ignores."scripts/*" = ["T20"]
|
per-file-ignores."scripts/*" = ["T20"]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
|
|||||||
+10
-9
@@ -16,15 +16,16 @@ Environment:
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import contextlib
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
from contextlib import suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Import devutil from scripts/fastapi-vue (not a package, so we adjust sys.path)
|
# Import devutil from scripts/fastapi-vue (not a package, so we adjust sys.path)
|
||||||
sys.path.insert(0, str(Path(__file__).with_name("fastapi-vue")))
|
sys.path.insert(0, str(Path(__file__).with_name("fastapi-vue")))
|
||||||
from devutil import ( # type: ignore[import-not-found]
|
from devutil import ( # type: ignore[import-not-found]
|
||||||
ProcessGroup,
|
ProcessGroup,
|
||||||
|
check_ports_free,
|
||||||
logger,
|
logger,
|
||||||
ready,
|
ready,
|
||||||
setup_vite,
|
setup_vite,
|
||||||
@@ -33,7 +34,9 @@ from devutil import ( # type: ignore[import-not-found]
|
|||||||
from cista import config
|
from cista import config
|
||||||
from cista.serve import parse_listen
|
from cista.serve import parse_listen
|
||||||
|
|
||||||
|
DEFAULT_VITE_PORT = 8989
|
||||||
DEFAULT_BACKEND_PORT = 8999
|
DEFAULT_BACKEND_PORT = 8999
|
||||||
|
HEALTH = "/api/health?from=devserver.py"
|
||||||
|
|
||||||
|
|
||||||
def setup_sanic_backend(
|
def setup_sanic_backend(
|
||||||
@@ -64,7 +67,7 @@ async def run_devserver(
|
|||||||
logger.warning("Frontend source not found at %s", front)
|
logger.warning("Frontend source not found at %s", front)
|
||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
|
|
||||||
_frontend_url, npm_install, vite = setup_vite(frontend or "")
|
frontend_url, npm_install, vite = setup_vite(frontend or "", DEFAULT_VITE_PORT)
|
||||||
backend_url, sanic_cmd = setup_sanic_backend(backend, extra_args)
|
backend_url, sanic_cmd = setup_sanic_backend(backend, extra_args)
|
||||||
|
|
||||||
# Tell vite where to proxy API requests
|
# Tell vite where to proxy API requests
|
||||||
@@ -72,19 +75,17 @@ async def run_devserver(
|
|||||||
|
|
||||||
async with ProcessGroup() as pg:
|
async with ProcessGroup() as pg:
|
||||||
install_proc = await pg.spawn(*npm_install, cwd=str(front))
|
install_proc = await pg.spawn(*npm_install, cwd=str(front))
|
||||||
await asyncio.sleep(0.2) # reduce message overlap
|
await check_ports_free(frontend_url, backend_url)
|
||||||
await pg.spawn(*sanic_cmd, cwd=str(reporoot))
|
await pg.spawn(*sanic_cmd, cwd=str(reporoot))
|
||||||
|
|
||||||
# Wait for both install and backend to be ready
|
# Wait for dependencies to be installed and backend to accept requests
|
||||||
async with asyncio.TaskGroup() as tg:
|
await pg.wait(install_proc, ready(backend_url, path=HEALTH))
|
||||||
tg.create_task(pg.wait(install_proc))
|
|
||||||
tg.create_task(ready(backend_url, path="/api/health?from=devserver.py"))
|
|
||||||
|
|
||||||
# Start Vite dev server (ProcessGroup waits for any exit, then terminates others)
|
# Start Vite dev server (ProcessGroup waits for any exit, then terminates others)
|
||||||
await pg.spawn(*vite, cwd=str(front))
|
await pg.spawn(*vite, cwd=str(front))
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Run Vite and Cista (Sanic) development servers",
|
description="Run Vite and Cista (Sanic) development servers",
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
@@ -102,7 +103,7 @@ def main():
|
|||||||
help="Cista backend endpoint (default: from config, or :8999)",
|
help="Cista backend endpoint (default: from config, or :8999)",
|
||||||
)
|
)
|
||||||
args, unknown = parser.parse_known_args()
|
args, unknown = parser.parse_known_args()
|
||||||
with contextlib.suppress(KeyboardInterrupt):
|
with suppress(KeyboardInterrupt):
|
||||||
asyncio.run(run_devserver(args.listen, args.backend, unknown))
|
asyncio.run(run_devserver(args.listen, args.backend, unknown))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
"""Hatch build hook for building Vue frontend during package build."""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from hatchling.builders.hooks.plugin.interface import (
|
|
||||||
BuildHookInterface, # type: ignore[import-not-found]
|
|
||||||
)
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent))
|
|
||||||
from buildutil import build
|
|
||||||
|
|
||||||
|
|
||||||
class CustomBuildHook(BuildHookInterface):
|
|
||||||
def initialize(self, version, build_data):
|
|
||||||
super().initialize(version, build_data)
|
|
||||||
build("frontend")
|
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
"""Hatch build hook for building Vue frontend during package build."""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from hatchling.builders.hooks.plugin.interface import BuildHookInterface
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent))
|
||||||
|
from buildutil import build
|
||||||
|
|
||||||
|
|
||||||
|
class CustomBuildHook(BuildHookInterface): # type: ignore[misc]
|
||||||
|
"""Hatch build hook that builds Vue frontend during package build."""
|
||||||
|
|
||||||
|
def initialize(self, version: str, build_data: dict) -> None: # type: ignore[override]
|
||||||
|
"""Build frontend before package is built."""
|
||||||
|
super().initialize(version, build_data)
|
||||||
|
build("frontend")
|
||||||
@@ -7,13 +7,15 @@ import shutil
|
|||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
MIN_NODE_VERSION = 20
|
||||||
|
|
||||||
|
|
||||||
class _PrefixFormatter(logging.Formatter):
|
class _PrefixFormatter(logging.Formatter):
|
||||||
"""Formatter that adds prefix based on log level."""
|
"""Formatter that adds prefix based on log level."""
|
||||||
|
|
||||||
def format(self, record: logging.LogRecord) -> str:
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
if record.levelno >= logging.WARNING:
|
if record.levelno >= logging.WARNING:
|
||||||
return f"┃ ⚠️ {record.getMessage()}"
|
return f"⚠️ {record.getMessage()}"
|
||||||
return record.getMessage()
|
return record.getMessage()
|
||||||
|
|
||||||
|
|
||||||
@@ -41,74 +43,108 @@ def _check_node_version(node_path: str) -> None:
|
|||||||
match = re.match(r"v(\d+)", version_str)
|
match = re.match(r"v(\d+)", version_str)
|
||||||
if match:
|
if match:
|
||||||
major_version = int(match.group(1))
|
major_version = int(match.group(1))
|
||||||
if major_version >= 20:
|
if major_version >= MIN_NODE_VERSION:
|
||||||
return
|
return
|
||||||
raise RuntimeError(
|
msg = f"Node.js {version_str} found, but v{MIN_NODE_VERSION}+ required"
|
||||||
f"Node.js {version_str} found, but v20+ required (install with nvm)"
|
raise RuntimeError(msg)
|
||||||
)
|
|
||||||
except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
|
except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
|
||||||
pass
|
pass
|
||||||
raise RuntimeError("Could not determine Node.js version")
|
msg = "Could not determine Node.js version"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_npm_runtime(tool: str) -> bool:
|
||||||
|
"""Validate npm runtime by checking Node.js version. Returns True if valid."""
|
||||||
|
node_path = shutil.which("node", path=str(Path(tool).parent))
|
||||||
|
if node_path is None:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
_check_node_version(node_path)
|
||||||
|
except RuntimeError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _find_runtime_from_env(options: list[str]) -> tuple[str, str] | None:
|
||||||
|
"""Find runtime specified by JS_RUNTIME environment variable."""
|
||||||
|
js_runtime_env = os.environ.get("JS_RUNTIME")
|
||||||
|
if not js_runtime_env:
|
||||||
|
return None
|
||||||
|
|
||||||
|
js_runtime = js_runtime_env
|
||||||
|
js_path = Path(js_runtime)
|
||||||
|
runtime_name = js_path.name
|
||||||
|
|
||||||
|
# Map node to npm
|
||||||
|
if runtime_name == "node":
|
||||||
|
runtime_name = "npm"
|
||||||
|
js_runtime = str(js_path.parent / "npm") if js_path.parent.name else "npm"
|
||||||
|
|
||||||
|
for option in options:
|
||||||
|
if option != runtime_name and not runtime_name.startswith(option):
|
||||||
|
continue
|
||||||
|
|
||||||
|
tool = shutil.which(js_runtime)
|
||||||
|
if tool is None:
|
||||||
|
msg = f"JS_RUNTIME={js_runtime_env}: {option} not found"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
if option == "npm":
|
||||||
|
node_path = shutil.which("node", path=str(Path(tool).parent))
|
||||||
|
if node_path is None:
|
||||||
|
msg = f"JS_RUNTIME={js_runtime_env}: node not found"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
_check_node_version(node_path)
|
||||||
|
|
||||||
|
return tool, option
|
||||||
|
|
||||||
|
msg = f"JS_RUNTIME={js_runtime_env} not recognized"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _auto_detect_runtime(options: list[str]) -> tuple[str, str]:
|
||||||
|
"""Auto-detect JavaScript runtime from available options."""
|
||||||
|
node_version_error: RuntimeError | None = None
|
||||||
|
|
||||||
|
for option in options:
|
||||||
|
tool = shutil.which(option)
|
||||||
|
if not tool:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if option == "npm" and not _validate_npm_runtime(tool):
|
||||||
|
try:
|
||||||
|
node_path = shutil.which("node", path=str(Path(tool).parent))
|
||||||
|
if node_path:
|
||||||
|
_check_node_version(node_path)
|
||||||
|
except RuntimeError as e:
|
||||||
|
node_version_error = e
|
||||||
|
continue
|
||||||
|
|
||||||
|
return tool, option
|
||||||
|
|
||||||
|
if node_version_error:
|
||||||
|
raise node_version_error
|
||||||
|
msg = "Node.js (v20+), Deno or Bun is required but none was found"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
|
||||||
def find_js_runtime() -> tuple[str, str]:
|
def find_js_runtime() -> tuple[str, str]:
|
||||||
"""Find a JavaScript runtime from JS_RUNTIME env or auto-detect.
|
"""Find a JavaScript runtime from JS_RUNTIME env or auto-detect.
|
||||||
|
|
||||||
Returns (tool_path, tool_name) where tool_name is "deno", "npm", or "bun".
|
Returns (tool_path, tool_name) where tool_name is "deno", "npm", or "bun".
|
||||||
Raises JSRuntimeError if no suitable runtime is found.
|
Raises RuntimeError if no suitable runtime is found.
|
||||||
"""
|
"""
|
||||||
options = ["npm", "deno", "bun"]
|
options = ["npm", "deno", "bun"]
|
||||||
node_version_error: RuntimeError | None = None
|
|
||||||
|
|
||||||
# Check for JS_RUNTIME environment variable
|
# Check for JS_RUNTIME environment variable
|
||||||
if js_runtime_env := os.environ.get("JS_RUNTIME"):
|
if result := _find_runtime_from_env(options):
|
||||||
js_runtime = js_runtime_env
|
return result
|
||||||
js_path = Path(js_runtime)
|
|
||||||
runtime_name = js_path.name
|
|
||||||
# Map node to npm
|
|
||||||
if runtime_name == "node":
|
|
||||||
runtime_name = "npm"
|
|
||||||
js_runtime = str(js_path.parent / "npm") if js_path.parent.name else "npm"
|
|
||||||
for option in options:
|
|
||||||
if option == runtime_name or runtime_name.startswith(option):
|
|
||||||
tool = shutil.which(js_runtime)
|
|
||||||
if tool is None:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"JS_RUNTIME={js_runtime_env}: {option} not found"
|
|
||||||
)
|
|
||||||
# Check Node.js version if using npm
|
|
||||||
if option == "npm":
|
|
||||||
node_path = shutil.which("node", path=str(Path(tool).parent))
|
|
||||||
if node_path is None:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"JS_RUNTIME={js_runtime_env}: node not found"
|
|
||||||
)
|
|
||||||
_check_node_version(node_path) # Raises on failure
|
|
||||||
return tool, option
|
|
||||||
raise RuntimeError(f"JS_RUNTIME={js_runtime_env} not recognized")
|
|
||||||
|
|
||||||
# Auto-detect
|
# Auto-detect
|
||||||
for option in options:
|
return _auto_detect_runtime(options)
|
||||||
if tool := shutil.which(option):
|
|
||||||
# Check Node.js version if using npm
|
|
||||||
if option == "npm":
|
|
||||||
node_path = shutil.which("node", path=str(Path(tool).parent))
|
|
||||||
if node_path is None:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
_check_node_version(node_path)
|
|
||||||
except RuntimeError as e:
|
|
||||||
node_version_error = e
|
|
||||||
continue # Try next runtime
|
|
||||||
return tool, option
|
|
||||||
|
|
||||||
# No runtime found - provide helpful error
|
|
||||||
if node_version_error:
|
|
||||||
raise node_version_error
|
|
||||||
raise RuntimeError("Node.js (v20+), Deno or Bun is required but none was found")
|
|
||||||
|
|
||||||
|
|
||||||
def find_build_tool():
|
def find_build_tool() -> tuple[list[str], list[str]]:
|
||||||
"""Find JavaScript runtime and construct install/build commands.
|
"""Find JavaScript runtime and construct install/build commands.
|
||||||
|
|
||||||
Returns (install_cmd, build_cmd) tuples of command lists.
|
Returns (install_cmd, build_cmd) tuples of command lists.
|
||||||
@@ -146,7 +182,7 @@ def find_dev_tool() -> list[str]:
|
|||||||
|
|
||||||
if name == "bun":
|
if name == "bun":
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Bun has a bug in WS proxying (https://github.com/oven-sh/bun/issues/9882). Consider using npm instead."
|
"Bun has a WS proxy bug (github.com/oven-sh/bun/issues/9882). Consider npm.",
|
||||||
)
|
)
|
||||||
|
|
||||||
return [tool, *dev_args[name]]
|
return [tool, *dev_args[name]]
|
||||||
@@ -179,10 +215,10 @@ def build(folder: str = "frontend") -> None:
|
|||||||
install_cmd, build_cmd = find_build_tool()
|
install_cmd, build_cmd = find_build_tool()
|
||||||
except RuntimeError as e:
|
except RuntimeError as e:
|
||||||
logger.warning(e)
|
logger.warning(e)
|
||||||
raise SystemExit(1) from e
|
raise SystemExit(1) from None
|
||||||
|
|
||||||
def run(cmd):
|
def run(cmd: list[str]) -> None:
|
||||||
display_cmd = [Path(cmd[0]).name, *cmd[1:]]
|
display_cmd = [Path(cmd[0]).stem, *cmd[1:]]
|
||||||
logger.info("### %s", " ".join(display_cmd))
|
logger.info("### %s", " ".join(display_cmd))
|
||||||
subprocess.run(cmd, check=True, cwd=folder) # noqa: S603
|
subprocess.run(cmd, check=True, cwd=folder) # noqa: S603
|
||||||
|
|
||||||
@@ -190,5 +226,5 @@ def build(folder: str = "frontend") -> None:
|
|||||||
run(install_cmd)
|
run(install_cmd)
|
||||||
logger.info("")
|
logger.info("")
|
||||||
run(build_cmd)
|
run(build_cmd)
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError:
|
||||||
raise SystemExit(1) from e
|
raise SystemExit(1) from None
|
||||||
|
|||||||
+119
-48
@@ -1,56 +1,79 @@
|
|||||||
"""Utilities meant for devserver script, used only in source repository with dev deps."""
|
"""Utilities meant for devserver script, used only in source repository with dev deps."""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import contextlib
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from contextlib import suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING, Any, Self
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from buildutil import find_dev_tool, find_install_tool, logger
|
from buildutil import find_dev_tool, find_install_tool, logger
|
||||||
from fastapi_vue.hostutil import parse_endpoint
|
from fastapi_vue.hostutil import parse_endpoint
|
||||||
|
|
||||||
DEFAULT_VITE_PORT = 8989
|
if TYPE_CHECKING:
|
||||||
DEFAULT_BACKEND_PORT = 8999
|
from collections.abc import Coroutine
|
||||||
|
|
||||||
|
|
||||||
class ProcessGroup:
|
class ProcessGroup:
|
||||||
"""Manage async subprocesses with automatic cleanup, like TaskGroup for processes."""
|
"""Manage async subprocesses with automatic cleanup, like TaskGroup for processes."""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self) -> None:
|
||||||
|
"""Initialize empty process tracking."""
|
||||||
self._procs: list[asyncio.subprocess.Process] = []
|
self._procs: list[asyncio.subprocess.Process] = []
|
||||||
|
self._cmds: dict[int, str] = {} # pid -> command name
|
||||||
|
|
||||||
async def spawn(
|
async def spawn(
|
||||||
self, *cmd: str, cwd: str | None = None
|
self,
|
||||||
|
*cmd: str,
|
||||||
|
cwd: str | None = None,
|
||||||
) -> asyncio.subprocess.Process:
|
) -> asyncio.subprocess.Process:
|
||||||
"""Spawn a subprocess and track it."""
|
"""Spawn a subprocess and track it."""
|
||||||
logger.info(">>> %s", " ".join([Path(cmd[0]).name, *cmd[1:]]))
|
cmd_name = Path(cmd[0]).stem
|
||||||
|
logger.info(">>> %s", " ".join([cmd_name, *cmd[1:]]))
|
||||||
proc = await asyncio.create_subprocess_exec(*cmd, cwd=cwd)
|
proc = await asyncio.create_subprocess_exec(*cmd, cwd=cwd)
|
||||||
self._procs.append(proc)
|
self._procs.append(proc)
|
||||||
|
self._cmds[proc.pid] = cmd_name
|
||||||
return proc
|
return proc
|
||||||
|
|
||||||
async def wait(self, proc: asyncio.subprocess.Process) -> None:
|
async def wait(
|
||||||
"""Wait for a process to complete, raise SystemExit(1) on failure."""
|
self,
|
||||||
if await proc.wait() != 0:
|
*waitables: "asyncio.subprocess.Process | Coroutine[Any, Any, Any]",
|
||||||
logger.warning("Command failed")
|
) -> None:
|
||||||
raise SystemExit(1)
|
"""Wait for processes/coroutines to complete, raise SystemExit on failure."""
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def wait_proc(proc: asyncio.subprocess.Process) -> None:
|
||||||
|
returncode = await proc.wait()
|
||||||
|
if returncode != 0:
|
||||||
|
cmd_name = self._cmds.get(proc.pid, "unknown")
|
||||||
|
raise subprocess.CalledProcessError(returncode, cmd_name)
|
||||||
|
|
||||||
|
tasks = [
|
||||||
|
wait_proc(w) if isinstance(w, asyncio.subprocess.Process) else w
|
||||||
|
for w in waitables
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
await asyncio.gather(*tasks)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
logger.warning("%s failed with exit status %d", e.cmd, e.returncode)
|
||||||
|
raise SystemExit(1) from None
|
||||||
|
|
||||||
|
async def __aenter__(self) -> Self:
|
||||||
|
"""Enter the async context manager."""
|
||||||
return self
|
return self
|
||||||
|
|
||||||
async def __aexit__(self, exc_type, *_):
|
async def __aexit__(self, exc_type: type[BaseException] | None, *_: object) -> None:
|
||||||
"""Wait for one process to exit, terminate others, then wait for all."""
|
"""Wait for one process to exit, terminate others, then wait for all."""
|
||||||
cleanup_task = asyncio.create_task(self._cleanup())
|
await self._cleanup(immediate=exc_type is not None)
|
||||||
try:
|
|
||||||
await asyncio.shield(cleanup_task)
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
# Shield was cancelled but cleanup_task continues - wait for it
|
|
||||||
await cleanup_task
|
|
||||||
|
|
||||||
async def _cleanup(self):
|
async def _cleanup(self, *, immediate: bool = False) -> None:
|
||||||
running = [p for p in self._procs if p.returncode is None]
|
running = [p for p in self._procs if p.returncode is None]
|
||||||
if not running:
|
if not running:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if not immediate:
|
||||||
# Wait for any one process to exit
|
# Wait for any one process to exit
|
||||||
|
with suppress(asyncio.CancelledError):
|
||||||
await asyncio.wait(
|
await asyncio.wait(
|
||||||
[asyncio.create_task(p.wait()) for p in running],
|
[asyncio.create_task(p.wait()) for p in running],
|
||||||
return_when=asyncio.FIRST_COMPLETED,
|
return_when=asyncio.FIRST_COMPLETED,
|
||||||
@@ -59,53 +82,75 @@ class ProcessGroup:
|
|||||||
# Terminate remaining processes
|
# Terminate remaining processes
|
||||||
for p in self._procs:
|
for p in self._procs:
|
||||||
if p.returncode is None:
|
if p.returncode is None:
|
||||||
with contextlib.suppress(ProcessLookupError):
|
with suppress(ProcessLookupError):
|
||||||
p.terminate()
|
p.terminate()
|
||||||
|
|
||||||
# Wait for all to finish (with overall timeout)
|
# Wait for all to finish (with overall timeout), shielded from cancellation
|
||||||
still_running = [p for p in self._procs if p.returncode is None]
|
still_running = [p for p in self._procs if p.returncode is None]
|
||||||
if still_running:
|
if still_running:
|
||||||
|
with suppress(asyncio.CancelledError):
|
||||||
try:
|
try:
|
||||||
await asyncio.wait_for(
|
await asyncio.shield(
|
||||||
|
asyncio.wait_for(
|
||||||
asyncio.gather(*[p.wait() for p in still_running]),
|
asyncio.gather(*[p.wait() for p in still_running]),
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
except TimeoutError:
|
except TimeoutError:
|
||||||
for p in self._procs:
|
for p in self._procs:
|
||||||
if p.returncode is None:
|
if p.returncode is None:
|
||||||
with contextlib.suppress(ProcessLookupError):
|
with suppress(ProcessLookupError):
|
||||||
p.kill()
|
p.kill()
|
||||||
await p.wait()
|
await p.wait()
|
||||||
|
|
||||||
|
|
||||||
async def ready(url: str, path: str = "") -> None:
|
async def check_ports_free(*urls: str) -> None:
|
||||||
|
"""Verify URLs are not responding (ports are free). Raise SystemExit if any respond."""
|
||||||
|
|
||||||
|
async def check(client: httpx.AsyncClient, url: str) -> None:
|
||||||
|
with suppress(httpx.RequestError):
|
||||||
|
res = await client.get(url, timeout=0.1)
|
||||||
|
server = res.headers.get("server", "server")
|
||||||
|
logger.warning("Conflicting %s already running at %s", server, url)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
await asyncio.gather(*[check(client, url) for url in urls])
|
||||||
|
|
||||||
|
|
||||||
|
async def ready(url: str, path: str = "", max_attempts: int = 50) -> None:
|
||||||
"""Wait for the server to be ready by polling an endpoint.
|
"""Wait for the server to be ready by polling an endpoint.
|
||||||
|
|
||||||
|
Use empty path to disable the check and make this return immediately.
|
||||||
Raises SystemExit(1) if server doesn't start in time.
|
Raises SystemExit(1) if server doesn't start in time.
|
||||||
"""
|
"""
|
||||||
max_attempts = 50
|
if not path:
|
||||||
full_url = f"{url}{path}"
|
return
|
||||||
|
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
for attempt in range(max_attempts):
|
for attempt in range(max_attempts):
|
||||||
try:
|
try:
|
||||||
await client.get(full_url, timeout=1.0)
|
await client.get(f"{url}{path}", timeout=1.0)
|
||||||
logger.info("✓ Backend ready!")
|
except httpx.RequestError:
|
||||||
return
|
|
||||||
except httpx.RequestError as e:
|
|
||||||
if attempt == max_attempts - 1:
|
if attempt == max_attempts - 1:
|
||||||
logger.warning("Backend didn't start in time")
|
logger.warning("Backend didn't start in time")
|
||||||
raise SystemExit(1) from e
|
raise SystemExit(1) from None
|
||||||
await asyncio.sleep(0.1)
|
await asyncio.sleep(0.1)
|
||||||
|
else:
|
||||||
|
logger.info("✓ Backend ready!")
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
def setup_vite(endpoint: str) -> tuple[str, list[str], list[str]]:
|
def setup_vite(
|
||||||
|
endpoint: str,
|
||||||
|
default_port: int = 5173,
|
||||||
|
) -> tuple[str, list[str], list[str]]:
|
||||||
"""Parse frontend endpoint and build commands.
|
"""Parse frontend endpoint and build commands.
|
||||||
|
|
||||||
Returns (url, install_cmd, dev_cmd).
|
Returns (url, install_cmd, dev_cmd).
|
||||||
Raises SystemExit(1) on invalid config.
|
Raises SystemExit(1) on invalid config.
|
||||||
"""
|
"""
|
||||||
endpoints = parse_endpoint(endpoint, DEFAULT_VITE_PORT)
|
endpoints = parse_endpoint(endpoint, default_port)
|
||||||
|
|
||||||
if "uds" in endpoints[0]:
|
if "uds" in endpoints[0]:
|
||||||
logger.warning("Unix sockets not supported with vite devserver")
|
logger.warning("Unix sockets not supported with vite devserver")
|
||||||
@@ -118,18 +163,53 @@ def setup_vite(endpoint: str) -> tuple[str, list[str], list[str]]:
|
|||||||
dev_cmd = find_dev_tool()
|
dev_cmd = find_dev_tool()
|
||||||
if host != "localhost":
|
if host != "localhost":
|
||||||
dev_cmd.append("--host" if len(endpoints) > 1 else f"--host={host}")
|
dev_cmd.append("--host" if len(endpoints) > 1 else f"--host={host}")
|
||||||
if port != 5173:
|
|
||||||
dev_cmd.append(f"--port={port}")
|
dev_cmd.append(f"--port={port}")
|
||||||
|
|
||||||
return f"http://{host}:{port}", install_cmd, dev_cmd
|
return f"http://{host}:{port}", install_cmd, dev_cmd
|
||||||
|
|
||||||
|
|
||||||
def setup_fastapi(
|
def setup_fastapi(
|
||||||
endpoint: str, module: str, default_port: int = DEFAULT_BACKEND_PORT
|
endpoint: str,
|
||||||
|
module: str,
|
||||||
|
default_port: int = 8000,
|
||||||
) -> tuple[str, list[str]]:
|
) -> tuple[str, list[str]]:
|
||||||
"""Parse backend endpoint and build fastapi dev command.
|
"""Parse backend endpoint and build uvicorn command.
|
||||||
|
|
||||||
Returns (url, cmd).
|
Returns (url, uvicorn_cmd).
|
||||||
|
Raises SystemExit(1) on invalid config.
|
||||||
|
"""
|
||||||
|
endpoints = parse_endpoint(endpoint, default_port)
|
||||||
|
|
||||||
|
if "uds" in endpoints[0]:
|
||||||
|
logger.warning("Unix sockets not supported with vite devserver")
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
host = endpoints[0]["host"]
|
||||||
|
port = endpoints[0]["port"]
|
||||||
|
reload_dir = module.split(".", maxsplit=1)[0] # Don't reload on frontend changes
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"uvicorn",
|
||||||
|
module,
|
||||||
|
f"--host={host}",
|
||||||
|
f"--port={port}",
|
||||||
|
"--reload",
|
||||||
|
f"--reload-dir={reload_dir}",
|
||||||
|
"--forwarded-allow-ips=*",
|
||||||
|
]
|
||||||
|
return f"http://{host}:{port}", cmd
|
||||||
|
|
||||||
|
|
||||||
|
def setup_cli(
|
||||||
|
cli: str,
|
||||||
|
endpoint: str,
|
||||||
|
default_port: int = 8000,
|
||||||
|
) -> tuple[str, list[str]]:
|
||||||
|
"""Parse backend endpoint and build CLI command.
|
||||||
|
|
||||||
|
Returns (url, cli_cmd).
|
||||||
Raises SystemExit(1) on invalid config.
|
Raises SystemExit(1) on invalid config.
|
||||||
"""
|
"""
|
||||||
endpoints = parse_endpoint(endpoint, default_port)
|
endpoints = parse_endpoint(endpoint, default_port)
|
||||||
@@ -141,14 +221,5 @@ def setup_fastapi(
|
|||||||
host = endpoints[0]["host"]
|
host = endpoints[0]["host"]
|
||||||
port = endpoints[0]["port"]
|
port = endpoints[0]["port"]
|
||||||
|
|
||||||
cmd = [
|
cmd = [cli, f"--listen={host}:{port}"]
|
||||||
"fastapi",
|
|
||||||
"dev",
|
|
||||||
"--entrypoint",
|
|
||||||
module,
|
|
||||||
"--host",
|
|
||||||
host,
|
|
||||||
"--port",
|
|
||||||
str(port),
|
|
||||||
]
|
|
||||||
return f"http://{host}:{port}", cmd
|
return f"http://{host}:{port}", cmd
|
||||||
|
|||||||
@@ -0,0 +1,220 @@
|
|||||||
|
from http.cookies import SimpleCookie
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from sanic import Sanic
|
||||||
|
|
||||||
|
from cista import auth, config
|
||||||
|
from cista.app import use_session
|
||||||
|
from cista.auth import bp as auth_bp
|
||||||
|
|
||||||
|
|
||||||
|
def _set_cookie_headers(response) -> list[str]:
|
||||||
|
return list(response.headers.get_list("set-cookie"))
|
||||||
|
|
||||||
|
|
||||||
|
def _cookie_header(response, name: str = "cista") -> dict[str, str]:
|
||||||
|
for header in _set_cookie_headers(response):
|
||||||
|
cookie = SimpleCookie()
|
||||||
|
cookie.load(header)
|
||||||
|
morsel = cookie.get(name)
|
||||||
|
if morsel is not None and morsel.value:
|
||||||
|
return {"Cookie": f"{name}={morsel.value}"}
|
||||||
|
raise AssertionError(f"response did not set cookie {name!r}")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def setup_auth_config(tmp_path: Path):
|
||||||
|
alice = config.User()
|
||||||
|
auth.set_password(alice, "secret")
|
||||||
|
admin = config.User(privileged=True)
|
||||||
|
auth.set_password(admin, "admin-secret")
|
||||||
|
config.config = config.Config(
|
||||||
|
path=tmp_path,
|
||||||
|
listen=":0",
|
||||||
|
public=False,
|
||||||
|
users={"alice": alice, "admin": admin},
|
||||||
|
)
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture()
|
||||||
|
async def client(setup_auth_config: Path):
|
||||||
|
app = Sanic(f"auth-builtins-test-{uuid4().hex}", strict_slashes=True)
|
||||||
|
|
||||||
|
@app.on_request
|
||||||
|
async def load_auth_context(request):
|
||||||
|
await use_session(request)
|
||||||
|
|
||||||
|
app.blueprint(auth_bp)
|
||||||
|
yield app.asgi_client
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_restricted_page_renders_login_form_when_logged_out(client):
|
||||||
|
_, res = await client.get("/auth/restricted/")
|
||||||
|
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert "Authentication Required" in res.text
|
||||||
|
assert "Username:" in res.text
|
||||||
|
assert "Password:" in res.text
|
||||||
|
assert "/auth/login" in res.text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_restricted_page_with_invalid_session_clears_cookie(client):
|
||||||
|
_, res = await client.get(
|
||||||
|
"/auth/restricted/",
|
||||||
|
headers={"Cookie": "cista=missing-session"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert "Authentication Required" in res.text
|
||||||
|
assert any("cista=" in header.lower() for header in _set_cookie_headers(res))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_json_login_sets_session_cookie_and_allows_session_authenticated_api_access(
|
||||||
|
client,
|
||||||
|
):
|
||||||
|
_, res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
json={"username": "alice", "password": "secret"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert res.json == {"data": {"username": "alice", "privileged": False}}
|
||||||
|
|
||||||
|
session_cookie = _cookie_header(res)
|
||||||
|
|
||||||
|
_, tokens_res = await client.get("/auth/tokens", headers=session_cookie)
|
||||||
|
assert tokens_res.status_code == 200
|
||||||
|
assert tokens_res.json == {"tokens": []}
|
||||||
|
|
||||||
|
_, restricted_res = await client.get("/auth/restricted/", headers=session_cookie)
|
||||||
|
assert restricted_res.status_code == 200
|
||||||
|
assert "auth-success" in restricted_res.text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_json_login_rejects_missing_fields(client):
|
||||||
|
_, res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
json={"username": "alice"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 400
|
||||||
|
assert "Missing username or password" in res.json["message"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_json_login_rejects_invalid_password(client):
|
||||||
|
_, res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
json={"username": "alice", "password": "wrong"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 403
|
||||||
|
assert "Invalid password" in res.json["message"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_html_login_redirects_and_sets_flash_and_session_cookies(client):
|
||||||
|
_, res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
data={"username": "alice", "password": "secret"},
|
||||||
|
headers={"Accept": "text/html"},
|
||||||
|
follow_redirects=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 302
|
||||||
|
assert res.headers["location"] == "/"
|
||||||
|
headers = _set_cookie_headers(res)
|
||||||
|
assert any("cista=" in header.lower() for header in headers)
|
||||||
|
assert any("message=" in header.lower() for header in headers)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_logout_json_revokes_the_existing_session(client):
|
||||||
|
_, login_res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
json={"username": "alice", "password": "secret"},
|
||||||
|
)
|
||||||
|
session_cookie = _cookie_header(login_res)
|
||||||
|
|
||||||
|
_, logout_res = await client.post("/auth/api/logout", headers=session_cookie)
|
||||||
|
|
||||||
|
assert logout_res.status_code == 200
|
||||||
|
assert logout_res.json == {"message": "Logged out"}
|
||||||
|
assert any("cista=" in header.lower() for header in _set_cookie_headers(logout_res))
|
||||||
|
|
||||||
|
_, retry_res = await client.get("/auth/tokens", headers=session_cookie)
|
||||||
|
assert retry_res.status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_logout_without_session_reports_not_logged_in(client):
|
||||||
|
_, res = await client.post("/auth/api/logout")
|
||||||
|
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert res.json == {"message": "Not logged in"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_password_change_updates_credentials_and_reissues_session(client):
|
||||||
|
_, change_res = await client.post(
|
||||||
|
"/auth/password-change",
|
||||||
|
json={
|
||||||
|
"username": "alice",
|
||||||
|
"password": "secret",
|
||||||
|
"passwordChange": "fresh-secret",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert change_res.status_code == 200
|
||||||
|
assert change_res.json == {"message": "Password updated"}
|
||||||
|
|
||||||
|
session_cookie = _cookie_header(change_res)
|
||||||
|
_, tokens_res = await client.get("/auth/tokens", headers=session_cookie)
|
||||||
|
assert tokens_res.status_code == 200
|
||||||
|
|
||||||
|
_, old_login_res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
json={"username": "alice", "password": "secret"},
|
||||||
|
)
|
||||||
|
assert old_login_res.status_code == 403
|
||||||
|
|
||||||
|
_, new_login_res = await client.post(
|
||||||
|
"/auth/login",
|
||||||
|
json={"username": "alice", "password": "fresh-secret"},
|
||||||
|
)
|
||||||
|
assert new_login_res.status_code == 200
|
||||||
|
assert new_login_res.json == {"data": {"username": "alice", "privileged": False}}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_password_change_rejects_wrong_current_password(client):
|
||||||
|
_, res = await client.post(
|
||||||
|
"/auth/password-change",
|
||||||
|
json={
|
||||||
|
"username": "alice",
|
||||||
|
"password": "wrong",
|
||||||
|
"passwordChange": "fresh-secret",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 403
|
||||||
|
assert "Invalid password" in res.json["message"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_password_change_rejects_missing_fields(client):
|
||||||
|
_, res = await client.post(
|
||||||
|
"/auth/password-change",
|
||||||
|
json={"username": "alice", "password": "secret"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert res.status_code == 400
|
||||||
|
assert "Missing username, passwordChange or password" in res.json["message"]
|
||||||
@@ -3,11 +3,11 @@ import hashlib
|
|||||||
import hmac
|
import hmac
|
||||||
import struct
|
import struct
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from time import time
|
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
|
from Crypto.Hash import MD4
|
||||||
from sanic import Sanic
|
from sanic import Sanic
|
||||||
|
|
||||||
from cista import auth, config, session, watching
|
from cista import auth, config, session, watching
|
||||||
@@ -29,8 +29,6 @@ def _ntlm_type3(
|
|||||||
username: str, password: str, domain: str, challenge: bytes
|
username: str, password: str, domain: str, challenge: bytes
|
||||||
) -> dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""Build an NTLMv2 Type 3 message for testing."""
|
"""Build an NTLMv2 Type 3 message for testing."""
|
||||||
from Crypto.Hash import MD4
|
|
||||||
|
|
||||||
# NT hash
|
# NT hash
|
||||||
nt_hash = MD4.new(password.encode("utf-16le")).digest()
|
nt_hash = MD4.new(password.encode("utf-16le")).digest()
|
||||||
# NTLMv2 hash
|
# NTLMv2 hash
|
||||||
@@ -94,10 +92,7 @@ def _ntlm_type3(
|
|||||||
|
|
||||||
def _session_cookie_header(username: str) -> dict[str, str]:
|
def _session_cookie_header(username: str) -> dict[str, str]:
|
||||||
token = "test-" + username
|
token = "test-" + username
|
||||||
session._sessions[token] = {
|
session.put(token, username)
|
||||||
"exp": int(time()) + session.max_age,
|
|
||||||
"username": username,
|
|
||||||
}
|
|
||||||
return {"Cookie": f"cista={token}"}
|
return {"Cookie": f"cista={token}"}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -160,7 +160,7 @@ async def test_mkcol_windows_drive_path_stays_within_root(client, setup_storage:
|
|||||||
# Either created inside the storage root (201) or sanitised away (400/404).
|
# Either created inside the storage root (201) or sanitised away (400/404).
|
||||||
# The important assertion: nothing was created outside the storage root.
|
# The important assertion: nothing was created outside the storage root.
|
||||||
assert not (Path("/c:") / "secret").exists()
|
assert not (Path("/c:") / "secret").exists()
|
||||||
assert not (Path("c:/secret")).exists()
|
assert not (Path("c:/secret")).exists() # noqa: ASYNC240
|
||||||
if res.status_code == 201:
|
if res.status_code == 201:
|
||||||
# Created safely inside tmp storage
|
# Created safely inside tmp storage
|
||||||
assert (setup_storage / "c:" / "secret").is_dir()
|
assert (setup_storage / "c:" / "secret").is_dir()
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path, PurePath
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import msgspec
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from sanic import Sanic
|
from sanic import Sanic
|
||||||
@@ -12,10 +13,6 @@ from cista.auth import bp as auth_bp
|
|||||||
|
|
||||||
|
|
||||||
def _persist_config():
|
def _persist_config():
|
||||||
from pathlib import PurePath
|
|
||||||
|
|
||||||
import msgspec
|
|
||||||
|
|
||||||
def enc_hook(obj):
|
def enc_hook(obj):
|
||||||
if isinstance(obj, PurePath):
|
if isinstance(obj, PurePath):
|
||||||
return obj.as_posix()
|
return obj.as_posix()
|
||||||
|
|||||||
Reference in New Issue
Block a user