From 0cf9c254e51853a722fe24edd43595adf75eb21c Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 15:32:49 +0000 Subject: [PATCH] Various build fixes, cleanup and details (#6) - Major memory usage reduction in video previews - Finally builds properly on Windows too Reviewed-on: https://git.zi.fi/Vasanko/cista-storage/pulls/6 --- cista/__main__.py | 5 ++++- cista/app.py | 9 ++++++++- cista/config.py | 34 ++++++++++++++++++++++++++-------- cista/preview.py | 23 +++++++++++++---------- cista/serve.py | 1 - frontend/.npmrc | 2 ++ frontend/package.json | 4 +++- frontend/vite.config.ts | 2 -- pyproject.toml | 1 + scripts/build-frontend.py | 24 +++++++++++++++++++++--- 10 files changed, 78 insertions(+), 27 deletions(-) create mode 100644 frontend/.npmrc diff --git a/cista/__main__.py b/cista/__main__.py index 407c60f..5822d15 100644 --- a/cista/__main__.py +++ b/cista/__main__.py @@ -1,3 +1,4 @@ +import os import sys from pathlib import Path @@ -61,6 +62,7 @@ def _main(): path = None _confdir(args) exists = config.conffile.exists() + print(config.conffile, exists) import_droppy = args["--import-droppy"] necessary_opts = exists or import_droppy or path if not necessary_opts: @@ -117,7 +119,8 @@ def _confdir(args): raise ValueError("Config path is not a directory") # Accidentally pointed to the db.toml, use parent confdir = confdir.parent - config.conffile = confdir / config.conffile.name + os.environ["CISTA_HOME"] = confdir.as_posix() + config.init_confdir() # Uses environ if available def _user(args): diff --git a/cista/app.py b/cista/app.py index e646fe4..a87cef1 100644 --- a/cista/app.py +++ b/cista/app.py @@ -3,6 +3,7 @@ import datetime import mimetypes import threading from concurrent.futures import ThreadPoolExecutor +from multiprocessing import cpu_count from pathlib import Path, PurePath, PurePosixPath from stat import S_IFDIR, S_IFREG from urllib.parse import unquote @@ -14,6 +15,7 @@ from blake3 import blake3 from sanic import Blueprint, Sanic, empty, raw, redirect from sanic.exceptions import Forbidden, NotFound from sanic.log import logger +from setproctitle import setproctitle from stream_zip import ZIP_AUTO, stream_zip from cista import auth, config, preview, session, watching @@ -30,11 +32,16 @@ app.blueprint(bp) app.exception(Exception)(handle_sanic_exception) +setproctitle("cista-main") + + @app.before_server_start async def main_start(app, loop): config.load_config() + setproctitle(f"cista {config.config.path.name}") + workers = max(2, min(8, cpu_count())) app.ctx.threadexec = ThreadPoolExecutor( - max_workers=8, thread_name_prefix="cista-ioworker" + max_workers=workers, thread_name_prefix="cista-ioworker" ) await watching.start(app, loop) diff --git a/cista/config.py b/cista/config.py index 3ecf9af..03fbfdd 100644 --- a/cista/config.py +++ b/cista/config.py @@ -1,7 +1,9 @@ from __future__ import annotations +import os import secrets import sys +from contextlib import suppress from functools import wraps from hashlib import sha256 from pathlib import Path, PurePath @@ -33,7 +35,23 @@ class Link(msgspec.Struct, omit_defaults=True): config = None -conffile = Path.home() / ".local/share/cista/db.toml" +conffile = None + + +def init_confdir(): + if p := os.environ.get("CISTA_HOME"): + home = Path(p) + else: + xdg = os.environ.get("XDG_CONFIG_HOME") + home = ( + Path(xdg).expanduser() / "cista" if xdg else Path.home() / ".config/cista" + ) + if not home.is_dir(): + home.mkdir(parents=True, exist_ok=True) + home.chmod(0o700) + + global conffile + conffile = home / "db.toml" def derived_secret(*params, len=8) -> bytes: @@ -61,8 +79,8 @@ def dec_hook(typ, obj): def config_update(modify): global config - if not conffile.exists(): - conffile.parent.mkdir(parents=True, exist_ok=True) + if conffile is None: + init_confdir() tmpname = conffile.with_suffix(".tmp") try: f = tmpname.open("xb") @@ -76,10 +94,6 @@ def config_update(modify): old = conffile.read_bytes() c = msgspec.toml.decode(old, type=Config, dec_hook=dec_hook) except FileNotFoundError: - # No existing config file, make sure we have a folder... - confdir = conffile.parent - confdir.mkdir(parents=True, exist_ok=True) - confdir.chmod(0o700) old = b"" c = None c = modify(c) @@ -92,7 +106,9 @@ def config_update(modify): f.write(new) f.close() if sys.platform == "win32": - conffile.unlink() # Windows doesn't support atomic replace + # Windows doesn't support atomic replace + with suppress(FileNotFoundError): + conffile.unlink() tmpname.rename(conffile) # Atomic replace except: f.close() @@ -120,6 +136,8 @@ def modifies_config(modify): def load_config(): global config + if conffile is None: + init_confdir() config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook) diff --git a/cista/preview.py b/cista/preview.py index e456599..a595057 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -1,4 +1,5 @@ import asyncio +import gc import io import mimetypes import urllib.parse @@ -17,6 +18,8 @@ from sanic.log import logger from cista import config from cista.util.filename import sanitize +DISPLAYMATRIX = av.stream.SideData.DISPLAYMATRIX + bp = Blueprint("preview", url_prefix="/preview") @@ -96,19 +99,19 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0): def process_video(path, *, maxsize, quality): with av.open(str(path)) as container: stream = container.streams.video[0] - rotation = ( - stream.side_data - and stream.side_data.get(av.stream.SideData.DISPLAYMATRIX) - or 0 - ) stream.codec_context.skip_frame = "NONKEY" + rot = stream.side_data and stream.side_data.get(DISPLAYMATRIX) or 0 container.seek(container.duration // 8) - frame = next(container.decode(stream)) - img = frame.to_image() + img = next(container.decode(stream)).to_image() + del stream img.thumbnail((maxsize, maxsize)) imgdata = io.BytesIO() - if rotation: - img = img.rotate(rotation, expand=True) + if rot: + img = img.rotate(rot, expand=True) img.save(imgdata, format="webp", quality=quality, method=4) - return imgdata.getvalue() + del img + ret = imgdata.getvalue() + del imgdata + gc.collect() + return ret diff --git a/cista/serve.py b/cista/serve.py index 24c9638..280ba31 100644 --- a/cista/serve.py +++ b/cista/serve.py @@ -26,7 +26,6 @@ def run(*, dev=False): motd=False, dev=dev, auto_reload=dev, - reload_dir={confdir}, access_log=True, ) # type: ignore if dev: diff --git a/frontend/.npmrc b/frontend/.npmrc new file mode 100644 index 0000000..09b35cd --- /dev/null +++ b/frontend/.npmrc @@ -0,0 +1,2 @@ +audit=false +fund=false diff --git a/frontend/package.json b/frontend/package.json index 4686e02..27894cd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,9 @@ "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore", "format": "prettier --write src/" }, + "engines": { + "node": ">=18.0.0" + }, "dependencies": { "@imengyu/vue3-context-menu": "^1.3.3", "@vueuse/core": "^10.4.1", @@ -21,7 +24,6 @@ "pinia": "^2.1.6", "pinia-plugin-persistedstate": "^3.2.0", "unplugin-vue-components": "^0.25.2", - "vite-plugin-rewrite-all": "^1.0.1", "vite-svg-loader": "^4.0.0", "vue": "^3.3.4", "vue-router": "^4.2.4" diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 2de1a87..834d0fd 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -4,7 +4,6 @@ import { defineConfig } from 'vite' import vue from '@vitejs/plugin-vue' // @ts-ignore -import pluginRewriteAll from 'vite-plugin-rewrite-all' import svgLoader from 'vite-svg-loader' import Components from 'unplugin-vue-components/vite' @@ -21,7 +20,6 @@ const dev_backend = { export default defineConfig({ plugins: [ vue(), - pluginRewriteAll(), svgLoader(), // import svg files Components(), // auto import components ], diff --git a/pyproject.toml b/pyproject.toml index 77d412c..ea6e73f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "pyjwt", "pymupdf", "sanic", + "setproctitle", "stream-zip", "tomli_w", ] diff --git a/scripts/build-frontend.py b/scripts/build-frontend.py index 10c1756..7f94070 100644 --- a/scripts/build-frontend.py +++ b/scripts/build-frontend.py @@ -1,5 +1,8 @@ # noqa: INP001 +import os +import shutil import subprocess +from sys import stderr from hatchling.builders.hooks.plugin.interface import BuildHookInterface @@ -7,6 +10,21 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface class CustomBuildHook(BuildHookInterface): def initialize(self, version, build_data): super().initialize(version, build_data) - print("Building Cista frontend...") - subprocess.run("npm install --prefix frontend".split(" "), check=True) # noqa: S603 - subprocess.run("npm run build --prefix frontend".split(" "), check=True) # noqa: S603 + # A hack to stop building twice on run + if not build_data.get("force_include"): + return + stderr.write(">>> Building Cista frontend\n") + npm = shutil.which("npm") + if npm is None: + raise RuntimeError( + "NodeJS `npm` is required for building Cista but it was not found" + ) + # npm --prefix doesn't work on Windows, so we chdir instead + os.chdir("frontend") + try: + stderr.write("### npm install\n") + subprocess.run([npm, "install"], check=True) # noqa: S603 + stderr.write("\n### npm run build\n") + subprocess.run([npm, "run", "build"], check=True) # noqa: S603 + finally: + os.chdir("..")