Compare commits

..
17 Commits
Author SHA1 Message Date
LeoVasanko 0190bda853 Avoid useless check. 2026-04-24 00:04:58 +00:00
LeoVasanko fc48500412 Upload end of file first for large files (for preview to function). 2026-04-24 00:02:17 +00:00
LeoVasanko d31ad0b525 Avoid showing previews while file upload is in progress or the file is incomplete. This avoids 500 errors on server trying to generate the preview. 2026-04-23 19:34:16 +00:00
LeoVasanko da4bba95be Handle missing preview files with silent 404 2026-04-23 19:02:54 +00:00
LeoVasanko e07ab220cb Add cleaner access logging for HTTP requests and WebSockets 2026-04-23 18:06:22 +00:00
LeoVasanko 3da2f6e6c3 Don't show broken media previews during uploads, instead show a spinner indicating completion. 2026-04-23 17:40:03 +00:00
LeoVasanko ded7ce65bc Faster image previews by pyvips. 2026-04-23 17:19:24 +00:00
LeoVasanko 17550be698 Allow duplicate selected entries of the same name from different paths. 2026-04-23 17:01:03 +00:00
LeoVasanko 0b269aef7f Set avif=1 flag for SVT-AV1 when encoding a single image, for increased efficiency and less memory usage. 2026-04-23 16:57:27 +00:00
LeoVasanko 497de296f2 Consistent forwarding of host/origin/ua over HTTP and WS to Paskia. 2026-02-10 23:56:42 +00:00
LeoVasanko 134b216f4c Cleanup 2026-02-10 23:52:07 +00:00
LeoVasanko 06759b3c12 pytest deps, remove unused test group. 2026-02-10 23:46:44 +00:00
LeoVasanko c51552ea29 Cleaner WebSocket/Paskia proxying. 2026-02-10 23:42:13 +00:00
LeoVasanko 00645fc8ff Attempt to get allocated size on Windows which needs WINAPI. Fallback to file size if not possible. 2026-02-05 20:31:46 +00:00
LeoVasanko 760f7bc35d Fix vite argument passing on deno (fastapi-vue-setup upstream). 2026-02-05 20:13:03 +00:00
LeoVasanko 8480a73839 Fix --listen docopt parsing (only -l was working). 2026-02-05 20:01:26 +00:00
LeoVasanko 302ed684e7 Remove invalid Sanic kwarg when binding to all interfaces (not needed anyway). 2026-02-05 19:56:54 +00:00
15 changed files with 545 additions and 89 deletions
+5
View File
@@ -20,6 +20,11 @@ Experience Cista by visiting [Cista Demo](https://drop.zi.fi) for a test run and
We recommend using [UV](https://docs.astral.sh/uv/getting-started/installation/) to directly run Cista: We recommend using [UV](https://docs.astral.sh/uv/getting-started/installation/) to directly run Cista:
Try it out locally at http://localhost:8000 (serves the current directory):
```fish
uvx cista
```
Create an account: (otherwise the server is public for all) Create an account: (otherwise the server is public for all)
```fish ```fish
uvx cista --user yourname --privileged uvx cista --user yourname --privileged
+4 -2
View File
@@ -61,10 +61,12 @@ doc = """\
Usage: Usage:
cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>] cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
cista [-c <confdir>] --user <name> [--privileged] [--password] cista [-c <confdir>] --user <name> [--privileged] [--password]
cista --version
Options: Options:
-c CONFDIR Custom config directory -c CONFDIR Custom config directory
-l LISTEN-ADDR Listen on -l, --listen LISTEN-ADDR
Listen on
:8989 (localhost port, plain http) :8989 (localhost port, plain http)
<addr>:3000 (bind another address, port) <addr>:3000 (bind another address, port)
/path/to/unix.sock (unix socket) /path/to/unix.sock (unix socket)
@@ -117,7 +119,7 @@ def _main():
args = docopt(doc) args = docopt(doc)
if args["--user"]: if args["--user"]:
return _user(args) return _user(args)
listen = args["-l"] listen = args["--listen"]
# Validate arguments first # Validate arguments first
if args["<path>"]: if args["<path>"]:
path = Path(args["<path>"]).resolve() path = Path(args["<path>"]).resolve()
+26
View File
@@ -1,6 +1,7 @@
import asyncio import asyncio
import datetime import datetime
import mimetypes import mimetypes
import time
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from multiprocessing import cpu_count from multiprocessing import cpu_count
from pathlib import Path, PurePath, PurePosixPath from pathlib import Path, PurePath, PurePosixPath
@@ -19,11 +20,15 @@ from zstandard import ZstdCompressor
from cista import auth, config, preview, session, sso, watching from cista import auth, config, preview, session, sso, watching
from cista.api import bp from cista.api import bp
from cista.sanic_logging import configure_access_logging, format_access_log
from cista.sanic_logging import logger as access_logger
from cista.util.apphelpers import handle_sanic_exception from cista.util.apphelpers import handle_sanic_exception
# Workaround until Sanic PR #2824 is merged # Workaround until Sanic PR #2824 is merged
sanic.helpers._ENTITY_HEADERS = frozenset() sanic.helpers._ENTITY_HEADERS = frozenset()
configure_access_logging()
app = Sanic("cista", strict_slashes=True) app = Sanic("cista", strict_slashes=True)
# Register either SSO proxy or built-in auth routes based on PASKIA_BACKEND_URL # Register either SSO proxy or built-in auth routes based on PASKIA_BACKEND_URL
if sso.paskia_enabled(): if sso.paskia_enabled():
@@ -64,6 +69,7 @@ async def main_stop(app):
@app.on_request @app.on_request
async def use_session(req): async def use_session(req):
req.ctx._log_start = time.perf_counter()
req.ctx.session = session.get(req) req.ctx.session = session.get(req)
try: try:
req.ctx.username = req.ctx.session["username"] # type: ignore req.ctx.username = req.ctx.session["username"] # type: ignore
@@ -81,6 +87,26 @@ async def use_session(req):
raise Forbidden("Invalid origin: Cross-Site requests not permitted") raise Forbidden("Invalid origin: Cross-Site requests not permitted")
@app.on_response
async def log_access(req, res):
"""Log HTTP access in a clean single-line format."""
if req.headers.get("upgrade", "").lower() == "websocket":
return res
start = getattr(req.ctx, "_log_start", None)
duration_ms = (time.perf_counter() - start) * 1000 if start is not None else 0.0
client = req.ip or "-"
host = req.host or "-"
path = req.path
if req.query_string:
qs = req.query_string
if isinstance(qs, bytes):
qs = qs.decode(errors="replace")
path = f"{path}?{qs}"
line = format_access_log(client, res.status, req.method, host, path, duration_ms)
access_logger.info(line)
return res
@app.on_response @app.on_response
async def forward_sso_cookies(req, res): async def forward_sso_cookies(req, res):
"""Forward Set-Cookie headers from SSO validation to client.""" """Forward Set-Cookie headers from SSO validation to client."""
+55 -7
View File
@@ -97,7 +97,11 @@ async def preview(req, path):
quality = int(req.args.get("q", 60)) quality = int(req.args.get("q", 60))
rel = PurePosixPath(sanitize(unquote(path))) rel = PurePosixPath(sanitize(unquote(path)))
filepath = config.config.path / rel filepath = config.config.path / rel
stat = filepath.lstat() try:
stat = filepath.lstat()
except FileNotFoundError:
raise NotFound() from None
etag = config.derived_secret( etag = config.derived_secret(
"preview", rel, stat.st_mtime_ns, quality, maxsize, maxzoom "preview", rel, stat.st_mtime_ns, quality, maxsize, maxzoom
).hex() ).hex()
@@ -112,9 +116,6 @@ async def preview(req, path):
logger.debug(f"Preview cache hit: {rel}") logger.debug(f"Preview cache hit: {rel}")
return raw(cached.body, headers=cached.headers) return raw(cached.body, headers=cached.headers)
if not filepath.is_file():
raise NotFound("File not found")
# Generate preview # Generate preview
img = await asyncio.get_event_loop().run_in_executor( img = await asyncio.get_event_loop().run_in_executor(
req.app.ctx.threadexec, dispatch, filepath, quality, maxsize, maxzoom req.app.ctx.threadexec, dispatch, filepath, quality, maxsize, maxzoom
@@ -153,6 +154,44 @@ def dispatch(path, quality, maxsize, maxzoom):
def process_image(path, *, maxsize, quality): def process_image(path, *, maxsize, quality):
try:
return process_image_pyvips(path, maxsize=maxsize, quality=quality)
except Exception as e:
logger.debug("Falling back to Pillow preview for %s: %s", path.name, e)
return process_image_pillow(path, maxsize=maxsize, quality=quality)
def process_image_pyvips(path, *, maxsize, quality):
import pyvips
t_load = perf_counter()
img = pyvips.Image.new_from_file(str(path), access="sequential")
t_proc = perf_counter()
img = img.autorot()
scale = min(maxsize / img.width, maxsize / img.height, 1.0)
if scale < 1.0:
img = img.resize(scale)
t_save = perf_counter()
ret = img.write_to_buffer(".avif", Q=quality)
t_end = perf_counter()
load_ms = (t_proc - t_load) * 1000
proc_ms = (t_save - t_proc) * 1000
save_ms = (t_end - t_save) * 1000
logger.debug(
"Preview image %s via pyvips: load=%.1fms process=%.1fms save=%.1fms",
path.name,
load_ms,
proc_ms,
save_ms,
)
return ret
def process_image_pillow(path, *, maxsize, quality):
t_load = perf_counter() t_load = perf_counter()
with Image.open(path) as img: with Image.open(path) as img:
# Force decode to include I/O in load timing # Force decode to include I/O in load timing
@@ -168,7 +207,14 @@ def process_image(path, *, maxsize, quality):
# Save as AVIF # Save as AVIF
imgdata = io.BytesIO() imgdata = io.BytesIO()
t_save = perf_counter() t_save = perf_counter()
img.save(imgdata, format="avif", quality=quality, speed=10, max_threads=1) img.save(
imgdata,
format="avif",
quality=quality,
speed=10,
max_threads=1,
avif=1,
)
t_end = perf_counter() t_end = perf_counter()
ret = imgdata.getvalue() ret = imgdata.getvalue()
@@ -177,7 +223,7 @@ def process_image(path, *, maxsize, quality):
proc_ms = (t_save - t_proc) * 1000 proc_ms = (t_save - t_proc) * 1000
save_ms = (t_end - t_save) * 1000 save_ms = (t_end - t_save) * 1000
logger.debug( logger.debug(
"Preview image %s: load=%.1fms process=%.1fms save=%.1fms", "Preview image %s via Pillow: load=%.1fms process=%.1fms save=%.1fms",
path.name, path.name,
load_ms, load_ms,
proc_ms, proc_ms,
@@ -198,7 +244,9 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
t_load_end = perf_counter() t_load_end = perf_counter()
t_save_start = perf_counter() t_save_start = perf_counter()
ret = pix.pil_tobytes(format="avif", quality=quality, speed=10, max_threads=1) ret = pix.pil_tobytes(
format="avif", quality=quality, speed=10, max_threads=1, avif=1
)
t_save_end = perf_counter() t_save_end = perf_counter()
logger.debug( logger.debug(
+220
View File
@@ -0,0 +1,220 @@
"""Custom access logging middleware for Sanic."""
import logging
import sys
import unicodedata
from ipaddress import IPv6Address
logger = logging.getLogger("cista.access")
_RESET = "\033[0m"
_STATUS_INFO = "\033[32m" # 1xx (green)
_STATUS_OK = "\033[1;92m" # 2xx (bright green)
_STATUS_REDIRECT = "\033[32m" # 3xx (green)
_STATUS_CLIENT_ERR = "\033[0;31m" # 4xx (red)
_STATUS_SERVER_ERR = "\033[1;91m" # 5xx (bold bright red)
_METHOD_READ = "\033[0;34m" # GET, HEAD, OPTIONS (blue)
_METHOD_WRITE = "\033[1;94m" # POST, PUT, DELETE, PATCH (bold bright blue)
_HOST = "\033[38;5;242m" # hostname (dark grey)
_PATH = "\033[38;5;250m" # path (light grey)
_TIMING = "\033[38;5;242m" # timing (dark grey)
_WS_OPEN = "\033[1;93m" # WebSocket connect (bold bright yellow)
_WS_CLOSE = "\033[33m" # WebSocket disconnect (yellow)
_WS_STATUS = "\033[38;5;250m" # WebSocket close status (normal white)
def format_ipv6_network(ip: str) -> str:
"""Format IPv6 address to show only network part (first 64 bits)."""
try:
ip = ip.strip("[]")
if "%" in ip:
ip = ip.split("%")[0]
addr = IPv6Address(ip)
if addr.is_loopback:
return "::1"
if addr.is_unspecified:
return "::"
if addr.ipv4_mapped:
return str(addr.ipv4_mapped)
if addr.is_link_local:
return str(addr)
network_int = int(addr) >> 64
groups = []
for _ in range(4):
groups.insert(0, format(network_int & 0xFFFF, "x"))
network_int >>= 16
result = ":".join(groups) + "::"
return str(IPv6Address(result + "0")).removesuffix("::")
except Exception:
return ip
def format_client_ip(ip: str) -> str:
"""Format client IP, compressing IPv6 to network part only."""
if not ip or ip == "-":
return "-"
stripped = ip.strip("[]")
if ":" in stripped:
return format_ipv6_network(ip)
return ip
def status_color(status: int) -> str:
if status < 200:
return _STATUS_INFO
if status < 300:
return _STATUS_OK
if status < 400:
return _STATUS_REDIRECT
if status < 500:
return _STATUS_CLIENT_ERR
return _STATUS_SERVER_ERR
def method_color(method: str) -> str:
if method in ("GET", "HEAD", "OPTIONS"):
return _METHOD_READ
return _METHOD_WRITE
def format_duration_ms(duration_ms: float) -> str:
rounded_ms = round(duration_ms)
if rounded_ms < 2000:
return f"{rounded_ms}ms"
total_s = round(duration_ms / 1000)
if total_s < 60:
return f"{total_s}s"
if total_s <= 3600:
minutes, seconds = divmod(total_s, 60)
return f"{minutes}m{seconds}s"
hours, remainder = divmod(total_s, 3600)
minutes = round(remainder / 60)
if minutes == 60:
hours += 1
minutes = 0
return f"{hours}h{minutes}m"
def _display_width(text: str) -> int:
width = 0
for char in text:
width += 2 if unicodedata.east_asian_width(char) in {"F", "W"} else 1
return width
def _format_left(label: str) -> str:
return label[:19].ljust(19)
def _format_method_label(label: str, *, color: str | None = None) -> str:
color_value = _METHOD_WRITE if color is None else color
padding = max(0, 7 - _display_width(label))
return f"{color_value}{label}{' ' * padding}{_RESET}"
def format_access_log(
client: str, status: int, method: str, host: str, path: str, duration_ms: float
) -> str:
ip = _format_left(format_client_ip(client))
status_str = f"{status_color(status)}{str(status).rjust(3)}{_RESET}"
method_str = _format_method_label(method, color=method_color(method))
host_str = f"{_HOST}{host}{_RESET}"
path_str = f"{_PATH}{path}{_RESET}"
timing_str = f"{_TIMING}{format_duration_ms(duration_ms)}{_RESET}"
return f"{ip} {status_str} {method_str} {host_str}{path_str} {timing_str}"
_ws_counter = 1
def _next_ws_id() -> int:
global _ws_counter
ws_id = _ws_counter
_ws_counter += 1
return ws_id
def _format_ws_id(ws_id: int, *, bright: bool = False) -> str:
value = str(ws_id) if ws_id >= 100 else f"{ws_id:02d}"
color = _WS_OPEN if bright else _WS_CLOSE
return f"{color}{value.rjust(3)}{_RESET}"
def log_ws_open(request, extra: str | None = None) -> int:
"""Log WebSocket connection open. Returns connection ID for use in log_ws_close."""
ws_id = _next_ws_id()
client = request.ip or "-"
host = request.host or "-"
path = request.path
origin = request.headers.get("origin")
ip = _format_left(format_client_ip(client))
id_str = _format_ws_id(ws_id, bright=True)
origin_host = origin.split("://", 1)[-1] if origin else None
show_origin = origin_host and origin_host != host
method_str = _format_method_label("🔌", color=_WS_OPEN)
host_str = f"{_HOST}{host}{_RESET}"
path_str = f"{_PATH}{path}{_RESET}"
origin_str = f" {_RESET}from {_HOST}{origin_host}{_RESET}" if show_origin else ""
extra_str = f" {_TIMING}{extra}{_RESET}" if extra else ""
logger.info(
"%s %s %s %s%s%s",
ip,
id_str,
method_str,
host_str,
path_str,
origin_str + extra_str,
)
return ws_id
WS_CLOSE_CODES = {
1000: "ok",
1001: "going away",
1002: "protocol error",
1003: "unsupported",
1005: "no status",
1006: "abnormal",
1007: "invalid data",
1008: "policy violation",
1009: "too large",
1010: "extension required",
1011: "server error",
1012: "restarting",
1013: "try again",
1014: "bad gateway",
1015: "tls error",
}
def log_ws_close(ws_id: int, close_code: int | None, duration: float) -> None:
"""Log WebSocket connection close with duration and status."""
id_str = _format_ws_id(ws_id)
timing = format_duration_ms(duration * 1000)
if close_code is None:
code = "----"
status = "unknown"
else:
code = str(close_code)
status = WS_CLOSE_CODES.get(close_code, f"code {close_code}")
method_str = _format_method_label("closed", color=_TIMING)
status_str = f"{_WS_STATUS}{code} {status}{_RESET}"
timing_str = f"{_TIMING}{timing}{_RESET}"
logger.info("%s %s %s %s %s", " " * 19, id_str, method_str, status_str, timing_str)
def configure_access_logging() -> None:
"""Configure the cista.access logger to output to stderr."""
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(logging.Formatter("%(message)s"))
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.propagate = False
+2 -7
View File
@@ -27,7 +27,7 @@ def run(*, dev=False):
motd=False, motd=False,
dev=dev, dev=dev,
auto_reload=dev, auto_reload=dev,
access_log=True, access_log=False,
) # type: ignore ) # type: ignore
if dev: if dev:
Sanic.serve() Sanic.serve()
@@ -62,11 +62,6 @@ def parse_listen(listen):
return "http://localhost", {"unix": unix.as_posix()} return "http://localhost", {"unix": unix.as_posix()}
host, port = ep["host"], ep["port"] host, port = ep["host"], ep["port"]
# When binding all interfaces, use single_listener=False for Sanic
if len(endpoints) > 1: if len(endpoints) > 1:
return f"http://localhost:{port}", { return f"http://localhost:{port}", {"host": host, "port": port}
"host": host,
"port": port,
"single_listener": False,
}
return f"http://{host}:{port}", {"host": host, "port": port} return f"http://{host}:{port}", {"host": host, "port": port}
+18 -36
View File
@@ -15,7 +15,8 @@ import re
import httpx import httpx
import websockets import websockets
from sanic import Blueprint from sanic import Blueprint, json
from sanic import raw as raw_response
from sanic.exceptions import Forbidden, SanicException, Unauthorized from sanic.exceptions import Forbidden, SanicException, Unauthorized
from sanic.log import logger from sanic.log import logger
@@ -220,8 +221,6 @@ async def proxy_auth_request(request):
if key.lower() not in resp_hop_by_hop if key.lower() not in resp_hop_by_hop
] ]
from sanic import raw as raw_response
return raw_response( return raw_response(
raw_content, raw_content,
status=response.status_code, status=response.status_code,
@@ -231,35 +230,31 @@ async def proxy_auth_request(request):
except httpx.RequestError as e: except httpx.RequestError as e:
logger.error(f"Auth proxy request failed: {e}") logger.error(f"Auth proxy request failed: {e}")
from sanic import json
return json( return json(
{"detail": "Authentication service unavailable", "error": str(e)}, {"detail": "Authentication service unavailable"},
status=503, status=503,
) )
async def proxy_auth_websocket(request, ws): async def proxy_auth_websocket(request, ws):
"""Proxy a WebSocket connection to the auth backend.""" """Proxy a WebSocket connection to the auth backend."""
path = request.path url = f"ws{PASKIA_BACKEND_URL.removeprefix('http')}{request.path}"
query_string = request.query_string if request.query_string:
ws_backend = PASKIA_BACKEND_URL.replace("http://", "ws://").replace( url = f"{url}?{request.query_string}"
"https://", "wss://"
)
url = f"{ws_backend}{path}"
if query_string:
url = f"{url}?{query_string}"
additional_headers = {} additional_headers = {}
if "cookie" in request.headers: if "cookie" in request.headers:
additional_headers["cookie"] = request.headers["cookie"] additional_headers["cookie"] = request.headers["cookie"]
if "authorization" in request.headers: if "authorization" in request.headers:
additional_headers["authorization"] = request.headers["authorization"] additional_headers["authorization"] = request.headers["authorization"]
if "host" in request.headers:
additional_headers["host"] = request.headers["host"]
if "origin" in request.headers: if "origin" in request.headers:
additional_headers["origin"] = request.headers["origin"] additional_headers["origin"] = request.headers["origin"]
if "user-agent" in request.headers: if "user-agent" in request.headers:
additional_headers["user-agent"] = request.headers["user-agent"] additional_headers["user-agent"] = request.headers["user-agent"]
additional_headers["x-forwarded-for"] = request.ip additional_headers["x-forwarded-for"] = request.client_ip.strip("[]")
additional_headers["x-forwarded-host"] = request.host additional_headers["x-forwarded-host"] = request.host
additional_headers["x-forwarded-proto"] = request.scheme additional_headers["x-forwarded-proto"] = request.scheme
@@ -291,23 +286,20 @@ async def proxy_auth_websocket(request, ws):
logger.error(f"WebSocket proxy to {url} failed: {e}") logger.error(f"WebSocket proxy to {url} failed: {e}")
def _is_websocket_request(request) -> bool: # Blueprint for auth proxy routes (only registered when paskia_enabled())
"""Check if the request is a WebSocket upgrade request.""" bp = Blueprint("sso", url_prefix="/auth")
connection = request.headers.get("connection", "").lower()
upgrade = request.headers.get("upgrade", "").lower()
connection_tokens = [t.strip() for t in connection.split(",")]
return "upgrade" in connection_tokens and upgrade == "websocket"
async def _handle_websocket_upgrade(request): @bp.websocket("/ws/<path:path>")
"""Handle WebSocket upgrade and proxy the connection.""" async def auth_websocket_proxy(request, ws, path=""):
protocol = request.transport.get_protocol() """Proxy WebSocket connections to the auth backend."""
ws = await protocol.websocket_handshake(request, subprotocols=None)
await proxy_auth_websocket(request, ws) await proxy_auth_websocket(request, ws)
# Blueprint for auth proxy routes (only registered when paskia_enabled()) @bp.websocket("/ws/")
bp = Blueprint("sso", url_prefix="/auth") async def auth_websocket_proxy_root(request, ws):
"""Proxy root WebSocket connections to the auth backend."""
await proxy_auth_websocket(request, ws)
@bp.route( @bp.route(
@@ -315,20 +307,10 @@ bp = Blueprint("sso", url_prefix="/auth")
) )
async def auth_proxy(request, path=""): async def auth_proxy(request, path=""):
"""Proxy all auth requests to the auth backend.""" """Proxy all auth requests to the auth backend."""
if _is_websocket_request(request):
await _handle_websocket_upgrade(request)
from sanic import empty
return empty()
return await proxy_auth_request(request) return await proxy_auth_request(request)
@bp.route("/", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"]) @bp.route("/", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"])
async def auth_proxy_root(request): async def auth_proxy_root(request):
"""Proxy root auth requests to the auth backend.""" """Proxy root auth requests to the auth backend."""
if _is_websocket_request(request):
await _handle_websocket_upgrade(request)
from sanic import empty
return empty()
return await proxy_auth_request(request) return await proxy_auth_request(request)
+20
View File
@@ -1,3 +1,4 @@
import time
from functools import wraps from functools import wraps
import msgspec import msgspec
@@ -8,6 +9,7 @@ from sanic.response import raw, redirect
from cista import auth from cista import auth
from cista.protocol import ErrorMsg from cista.protocol import ErrorMsg
from cista.sanic_logging import log_ws_close, log_ws_open
def asend(ws, msg): def asend(ws, msg):
@@ -54,6 +56,10 @@ def websocket_wrapper(handler):
@wraps(handler) @wraps(handler)
async def wrapper(request, ws, *args, **kwargs): async def wrapper(request, ws, *args, **kwargs):
username = getattr(request.ctx, "username", None)
extra = username if username else None
start = time.perf_counter()
ws_id = log_ws_open(request, extra=extra)
try: try:
await auth.verify(request) await auth.verify(request)
await handler(request, ws, *args, **kwargs) await handler(request, ws, *args, **kwargs)
@@ -67,5 +73,19 @@ def websocket_wrapper(handler):
if not getattr(e, "quiet", False) or code == 500: if not getattr(e, "quiet", False) or code == 500:
logger.exception(f"{code} {e!r}") logger.exception(f"{code} {e!r}")
raise raise
finally:
duration = time.perf_counter() - start
close_code = None
try:
p = ws.ws_proto
if p.close_rcvd is not None:
close_code = p.close_rcvd.code
elif p.close_sent is not None:
close_code = p.close_sent.code
elif getattr(p, "close_code", None) is not None:
close_code = p.close_code
except AttributeError:
pass
log_ws_close(ws_id, close_code, duration)
return wrapper return wrapper
+32 -2
View File
@@ -17,6 +17,33 @@ from cista import config
from cista.fileio import fuid from cista.fileio import fuid
from cista.protocol import FileEntry, Space, UpdDel, UpdIns, UpdKeep from cista.protocol import FileEntry, Space, UpdDel, UpdIns, UpdKeep
# Platform-specific allocated size calculation
if sys.platform == "win32":
import ctypes
from ctypes import wintypes
kernel32 = ctypes.windll.kernel32
GetCompressedFileSizeW = kernel32.GetCompressedFileSizeW
GetCompressedFileSizeW.argtypes = [wintypes.LPCWSTR, ctypes.POINTER(wintypes.DWORD)]
GetCompressedFileSizeW.restype = wintypes.DWORD
INVALID_FILE_SIZE = 0xFFFFFFFF
def get_allocated_size(path: Path, st: stat_result) -> int:
"""Get actual disk allocation on Windows using GetCompressedFileSizeW."""
high = wintypes.DWORD()
low = GetCompressedFileSizeW(str(path), ctypes.byref(high))
if low == INVALID_FILE_SIZE and ctypes.get_last_error() != 0:
raise OSError(f"GetCompressedFileSizeW failed for {path}")
return (high.value << 32) + low
else:
def get_allocated_size(path: Path, st: stat_result) -> int:
"""Get actual disk allocation on Unix using st_blocks."""
# st_blocks is in 512-byte units
return st.st_blocks * 512
pubsub = {} pubsub = {}
sortkey = natsort_keygen(alg=ns.LOCALE) sortkey = natsort_keygen(alg=ns.LOCALE)
@@ -148,8 +175,11 @@ def walk(rel: PurePosixPath, stat: stat_result | None = None) -> list[FileEntry]
try: try:
st = stat or path.stat() st = stat or path.stat()
isfile = int(not S_ISDIR(st.st_mode)) isfile = int(not S_ISDIR(st.st_mode))
# st_blocks is in 512-byte units try:
allocated = st.st_blocks * 512 if isfile else 0 allocated = get_allocated_size(path, st) if isfile else 0
except Exception:
logger.exception(f"get_allocated_size failed for {path}")
allocated = st.st_size if isfile else 0
entry = FileEntry( entry = FileEntry(
level=len(rel.parts), level=len(rel.parts),
name=rel.name, name=rel.name,
+46 -6
View File
@@ -1,9 +1,13 @@
<template> <template>
<img v-if=preview() :src="`${doc.previewurl}?${quality}&t=${doc.mtime}`" alt=""> <div v-if=showProgress() class="preview-progress" aria-label="Preview pending">
<SpinnerIcon />
</div>
<img v-else-if=previewSrc :src="previewSrc" alt="">
<img v-else-if=doc.img :src=doc.url alt=""> <img v-else-if=doc.img :src=doc.url alt="">
<span v-else-if=doc.dir class="folder icon"></span> <span v-else-if=doc.dir class="folder icon"></span>
<div v-else-if=video() class="video-container"> <div v-else-if=video() class="video-container" :class="{ pending: !doc.complete }">
<video ref=vid :src=doc.url :poster=poster preload=none @play=onplay @pause=onpaused @ended=next @seeking=media!.play()></video> <video v-if=doc.complete ref=vid :src=doc.url :poster=previewSrc preload=none @play=onplay @pause=onpaused @ended=next @seeking=media!.play()></video>
<video v-else ref=vid :src=doc.url preload=none @play=onplay @pause=onpaused @ended=next @seeking=media!.play()></video>
<div class="play-overlay"><PlayIcon /></div> <div class="play-overlay"><PlayIcon /></div>
</div> </div>
<div v-else-if=audio() class="audio icon"> <div v-else-if=audio() class="audio icon">
@@ -16,16 +20,16 @@
<script setup lang=ts> <script setup lang=ts>
import { computed, ref } from 'vue' import { computed, ref } from 'vue'
import type { Doc } from '@/repositories/Document' import type { Doc } from '@/repositories/Document'
import { Play as PlayIcon } from '@/assets/svg' import { Play as PlayIcon, Spinner as SpinnerIcon } from '@/assets/svg'
const aud = ref<HTMLAudioElement | null>(null) const aud = ref<HTMLAudioElement | null>(null)
const vid = ref<HTMLVideoElement | null>(null) const vid = ref<HTMLVideoElement | null>(null)
const media = computed(() => aud.value || vid.value) const media = computed(() => aud.value || vid.value)
const poster = computed(() => `${props.doc.previewurl}?${props.quality}&t=${props.doc.mtime}`)
const props = defineProps<{ const props = defineProps<{
doc: Doc doc: Doc
quality: string quality: string
}>() }>()
const previewSrc = computed(() => props.doc.previewurl ? `${props.doc.previewurl}?${props.quality}&t=${props.doc.mtime}` : '')
const onplay = () => { const onplay = () => {
if (!media.value) return if (!media.value) return
@@ -37,6 +41,13 @@ const onpaused = () => {
media.value.controls = false media.value.controls = false
media.value.removeAttribute('data-playing') media.value.removeAttribute('data-playing')
} }
const applyPoster = (el: HTMLVideoElement) => {
if (props.doc.complete) {
el.poster = previewSrc.value
} else {
el.removeAttribute('poster')
}
}
let fscurrent: HTMLVideoElement | null = null let fscurrent: HTMLVideoElement | null = null
const next = () => { const next = () => {
if (!media.value) return if (!media.value) return
@@ -70,7 +81,7 @@ const next = () => {
if (!elem.paused) fscurrent.play() if (!elem.paused) fscurrent.play()
fscurrent = null fscurrent = null
elem.src = props.doc.url elem.src = props.doc.url
elem.poster = poster.value applyPoster(elem)
onpaused() onpaused()
}, {once: true}) }, {once: true})
} }
@@ -104,6 +115,7 @@ defineExpose({
const video = () => ['mkv', 'mp4', 'webm', 'mov', 'avi'].includes(props.doc.ext) const video = () => ['mkv', 'mp4', 'webm', 'mov', 'avi'].includes(props.doc.ext)
const audio = () => ['mp3', 'flac', 'ogg', 'aac'].includes(props.doc.ext) const audio = () => ['mp3', 'flac', 'ogg', 'aac'].includes(props.doc.ext)
const archive = () => ['zip', 'tar', 'gz', 'bz2', 'xz', '7z', 'rar'].includes(props.doc.ext) const archive = () => ['zip', 'tar', 'gz', 'bz2', 'xz', '7z', 'rar'].includes(props.doc.ext)
const showProgress = () => !props.doc.complete && (preview() || props.doc.img)
const preview = () => ( const preview = () => (
['bmp', 'ico', 'tif', 'tiff', 'heic', 'heif', 'pdf', 'epub', 'mobi'].includes(props.doc.ext) || ['bmp', 'ico', 'tif', 'tiff', 'heic', 'heif', 'pdf', 'epub', 'mobi'].includes(props.doc.ext) ||
props.doc.size > 500000 && props.doc.size > 500000 &&
@@ -120,6 +132,29 @@ img, embed, .icon, audio, video {
max-height: 100%; max-height: 100%;
border-radius: calc(.5em / 8); border-radius: calc(.5em / 8);
} }
.preview-progress {
display: flex;
align-items: center;
justify-content: center;
min-width: 50%;
max-width: 100%;
max-height: 100%;
aspect-ratio: 1;
}
.preview-progress :deep(svg) {
width: 4.5em;
height: 4.5em;
opacity: 0.8;
animation: media-preview-spin 0.9s linear infinite;
}
@keyframes media-preview-spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
.folder::before { .folder::before {
content: '📁'; content: '📁';
} }
@@ -175,9 +210,14 @@ img::before {
align-items: center; align-items: center;
justify-content: center; justify-content: center;
min-width: 50%; min-width: 50%;
min-height: 6em;
aspect-ratio: 16 / 9;
max-width: 100%; max-width: 100%;
max-height: 100%; max-height: 100%;
} }
.video-container.pending {
background: color-mix(in srgb, var(--header-bg) 55%, transparent);
}
.video-container video { .video-container video {
width: 100%; width: 100%;
height: 100%; height: 100%;
+105 -15
View File
@@ -131,6 +131,7 @@ const uploadCloudFiles = (files: CloudFile[]) => {
const cancelUploads = () => { const cancelUploads = () => {
upqueue = [] upqueue = []
blockQueue = []
statReset() statReset()
} }
@@ -162,16 +163,42 @@ setInterval(() => {
store.uprogress.statdur *= .9 store.uprogress.statdur *= .9
} }
}, 100) }, 100)
// Track uploaded bytes for each file to handle out-of-order uploads
const uploadedBytes = new Map<string, Set<number>>()
const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => { const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => {
if (name !== store.uprogress.filename) return // If stats have been reset if (name !== store.uprogress.filename) return // If stats have been reset
const now = Date.now() const now = Date.now()
store.uprogress.xfer = store.uprogress.filestart + end
store.uprogress.filepos = end // Track which bytes have been uploaded (using start to end range)
if (!uploadedBytes.has(name)) uploadedBytes.set(name, new Set())
const uploaded = uploadedBytes.get(name)!
const blockSize = 1 << 20
// Mark all bytes in this block as uploaded
for (let i = start; i < end; i += blockSize) {
uploaded.add(i)
}
// Calculate total uploaded bytes for progress
let totalUploaded = 0
for (let i = 0; i < size; i += blockSize) {
if (uploaded.has(i)) totalUploaded += blockSize
}
store.uprogress.xfer = store.uprogress.filestart + totalUploaded
store.uprogress.filepos = totalUploaded
store.uprogress.statbytes += end - start store.uprogress.statbytes += end - start
store.uprogress.statdur += now - store.uprogress.tlast store.uprogress.statdur += now - store.uprogress.tlast
store.uprogress.tlast = now store.uprogress.tlast = now
// File finished?
if (end === size) { // Check if file is fully uploaded by examining the block queue
const currentUpload = blockQueue[0]
if (!currentUpload) return
if (currentUpload.file.cloudName === name && currentUpload.blockIndex >= currentUpload.blocks.length) {
// All blocks for this file have been uploaded
uploadedBytes.delete(name) // Clean up tracking
store.uprogress.filestart += size store.uprogress.filestart += size
statNextFile() statNextFile()
if (++store.uprogress.fileidx >= store.uprogress.filecount) statReset() if (++store.uprogress.fileidx >= store.uprogress.filecount) statReset()
@@ -198,6 +225,42 @@ const statsAdd = (f: CloudFile[]) => {
} }
let upqueue = [] as CloudFile[] let upqueue = [] as CloudFile[]
// Helper function to get upload blocks for a file, prioritizing final 4 blocks if file >= 32 MiB
const getUploadBlocks = (file: CloudFile): {start: number, end: number}[] => {
const BLOCK_SIZE = 1 << 20 // 1 MiB
const MIN_SIZE_FOR_REORDER = 32 * BLOCK_SIZE // 32 MiB = 33554432 bytes
const FINAL_BLOCKS_COUNT = 2
const fileSize = file.file.size
const blocks: {start: number, end: number}[] = []
if (fileSize >= MIN_SIZE_FOR_REORDER) {
// File is large enough, prioritize final blocks
const finalBlocksStart = fileSize - (FINAL_BLOCKS_COUNT * BLOCK_SIZE)
// Add final blocks first
for (let i = 0; i < FINAL_BLOCKS_COUNT; i++) {
const start = finalBlocksStart + (i * BLOCK_SIZE)
const end = Math.min(start + BLOCK_SIZE, fileSize)
blocks.push({start, end})
}
// Add remaining blocks from beginning
for (let start = 0; start < finalBlocksStart; start += BLOCK_SIZE) {
const end = Math.min(start + BLOCK_SIZE, finalBlocksStart)
blocks.push({start, end})
}
} else {
// File is smaller, use sequential upload
for (let start = 0; start < fileSize; start += BLOCK_SIZE) {
const end = Math.min(start + BLOCK_SIZE, fileSize)
blocks.push({start, end})
}
}
return blocks
}
// TODO: Rewrite as WebSocket class // TODO: Rewrite as WebSocket class
const WSCreate = async () => await new Promise<WebSocket>(resolve => { const WSCreate = async () => await new Promise<WebSocket>(resolve => {
const ws = connect(uploadUrl, { const ws = connect(uploadUrl, {
@@ -235,31 +298,58 @@ const WSCreate = async () => await new Promise<WebSocket>(resolve => {
ws.send(data) ws.send(data)
} }
}) })
type BlockUpload = {
file: CloudFile
blocks: {start: number, end: number}[]
blockIndex: number
}
let blockQueue = [] as BlockUpload[]
const worker = async () => { const worker = async () => {
const ws = await WSCreate() const ws = await WSCreate()
while (upqueue.length) { while (blockQueue.length) {
const f = upqueue[0]! const upload = blockQueue[0]!
const start = f.cloudPos const f = upload.file
const end = Math.min(f.file.size, start + (1<<20)) const block = upload.blocks[upload.blockIndex]!
const control = { name: f.cloudName, size: f.file.size, start, end }
const data = f.file.slice(start, end) const control = { name: f.cloudName, size: f.file.size, start: block.start, end: block.end }
f.cloudPos = end const data = f.file.slice(block.start, block.end)
// Note: files may get modified during I/O // Note: files may get modified during I/O
// @ts-ignore FIXME proper WebSocket class, avoid attaching functions to WebSocket object // @ts-ignore FIXME proper WebSocket class, avoid attaching functions to WebSocket object
ws.sendMsg(control) ws.sendMsg(control)
// @ts-ignore // @ts-ignore
await ws.sendData(data) await ws.sendData(data)
if (f.cloudPos === f.file.size) upqueue.shift()
// Move to next block
upload.blockIndex++
if (upload.blockIndex >= upload.blocks.length) {
// File upload complete
blockQueue.shift()
}
} }
if (upqueue.length) startWorker() if (blockQueue.length) startWorker()
store.uprogress.status = "idle" store.uprogress.status = "idle"
workerRunning = false workerRunning = false
} }
let workerRunning: any = false let workerRunning: any = false
const startWorker = () => { const startWorker = () => {
if (workerRunning === false) workerRunning = setTimeout(() => { if (workerRunning === false) workerRunning = setTimeout(() => {
workerRunning = true // Convert new CloudFile entries to BlockUpload entries
worker() while (upqueue.length) {
const file = upqueue.shift()!
const blocks = getUploadBlocks(file)
blockQueue.push({ file, blocks, blockIndex: 0 })
}
if (blockQueue.length) {
workerRunning = true
worker()
} else {
workerRunning = false
}
}, 0) }, 0)
} }
+4
View File
@@ -59,6 +59,9 @@ export class Doc {
if (this.dir) return false if (this.dir) return false
return ['jpg', 'jpeg', 'png', 'gif', 'webp', 'avif', 'heic', 'heif', 'svg'].includes(this.ext) return ['jpg', 'jpeg', 'png', 'gif', 'webp', 'avif', 'heic', 'heif', 'svg'].includes(this.ext)
} }
get complete(): boolean {
return !this.ghost && (this.dir || this.size <= this.allocated)
}
get previewable(): boolean { get previewable(): boolean {
// Folders cannot be previewable // Folders cannot be previewable
if (this.dir) return false if (this.dir) return false
@@ -67,6 +70,7 @@ export class Doc {
return ['mp4', 'mkv', 'webm', 'ogg', 'mp3', 'flac', 'aac', 'pdf'].includes(this.ext) return ['mp4', 'mkv', 'webm', 'ogg', 'mp3', 'flac', 'aac', 'pdf'].includes(this.ext)
} }
get previewurl(): string { get previewurl(): string {
if (!this.complete || this.dir) return ''
return this.url.replace(/^\/files/, '/preview') return this.url.replace(/^\/files/, '/preview')
} }
get ext(): string { get ext(): string {
+2 -8
View File
@@ -378,22 +378,16 @@ export const useMainStore = defineStore('main', {
// What did we not select? // What did we not select?
for (const key of selected) if (!found.has(key)) ret.missing.add(key) for (const key of selected) if (!found.has(key)) ret.missing.add(key)
// Build a flat list including contents recursively // Build a flat list including contents recursively
const relnames = new Set<string>()
function add(rel: string, full: string, doc: Doc) {
if (!doc.dir && relnames.has(rel)) throw Error(`Multiple selections conflict for: ${rel}`)
relnames.add(rel)
ret.recursive.push([rel, full, doc])
}
for (const key of ret.keys) { for (const key of ret.keys) {
const base = ret.docs[key]! const base = ret.docs[key]!
const basepath = base.loc ? `${base.loc}/${base.name}` : base.name const basepath = base.loc ? `${base.loc}/${base.name}` : base.name
const nremove = base.loc.length const nremove = base.loc.length
add(base.name, basepath, base) ret.recursive.push([base.name, basepath, base])
for (const doc of docs) { for (const doc of docs) {
if (doc.loc === basepath || doc.loc.startsWith(basepath) && doc.loc[basepath.length] === '/') { if (doc.loc === basepath || doc.loc.startsWith(basepath) && doc.loc[basepath.length] === '/') {
const full = doc.loc ? `${doc.loc}/${doc.name}` : doc.name const full = doc.loc ? `${doc.loc}/${doc.name}` : doc.name
const rel = full.slice(nremove) const rel = full.slice(nremove)
add(rel, full, doc) ret.recursive.push([rel, full, doc])
} }
} }
} }
+5 -5
View File
@@ -41,10 +41,12 @@ dependencies = [
"pillow-heif>=1.1.0", "pillow-heif>=1.1.0",
"pyjwt>=2.10.1", "pyjwt>=2.10.1",
"pymupdf>=1.26.3", "pymupdf>=1.26.3",
"pyvips[binary]>=3.1.1",
"sanic>=25.12.0", "sanic>=25.12.0",
"setproctitle>=1.3.6", "setproctitle>=1.3.6",
"stream-zip>=0.0.83", "stream-zip>=0.0.83",
"tomli_w>=1.2.0", "tomli_w>=1.2.0",
"tracerite>=2.3.1",
"zstandard>=0.24.0", "zstandard>=0.24.0",
] ]
@@ -114,6 +116,7 @@ filterwarnings = [
] ]
[tool.ruff.lint] [tool.ruff.lint]
extend-select = ["E402"]
isort.known-first-party = ["cista"] isort.known-first-party = ["cista"]
per-file-ignores."tests/*" = ["S", "ANN", "D", "INP", "PLR2004"] per-file-ignores."tests/*" = ["S", "ANN", "D", "INP", "PLR2004"]
per-file-ignores."scripts/*" = ["T20"] per-file-ignores."scripts/*" = ["T20"]
@@ -121,16 +124,13 @@ per-file-ignores."scripts/*" = ["T20"]
[dependency-groups] [dependency-groups]
dev = [ dev = [
"pytest>=8.4.1", "pytest>=8.4.1",
"pytest-asyncio>=0.25.0",
"pytest-cov>=7.0.0",
"ruff>=0.8.0", "ruff>=0.8.0",
"mypy>=1.13.0", "mypy>=1.13.0",
"pre-commit>=4.0.0", "pre-commit>=4.0.0",
"httpx>=0.28.1", "httpx>=0.28.1",
] ]
test = [
"pytest>=8.4.1",
"pytest-cov>=6.0.0",
"pytest-asyncio>=0.25.0",
]
[tool.coverage.run] [tool.coverage.run]
source = ["cista"] source = ["cista"]
+1 -1
View File
@@ -134,7 +134,7 @@ def find_dev_tool() -> list[str]:
Raises RuntimeError if no runtime is found. Raises RuntimeError if no runtime is found.
""" """
dev_args = { dev_args = {
"deno": ("run", "dev", "--"), "deno": ("run", "-A", "npm:vite"),
"npm": ("--silent", "run", "dev", "--"), "npm": ("--silent", "run", "dev", "--"),
"bun": ("run", "dev", "--"), "bun": ("run", "dev", "--"),
} }