Various build fixes, cleanup and details (#6)

- Major memory usage reduction in video previews
- Finally builds properly on Windows too

Reviewed-on: #6

cista/__main__.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from pathlib import Path
 
@@ -61,6 +62,7 @@ def _main():
         path = None
     _confdir(args)
     exists = config.conffile.exists()
+    print(config.conffile, exists)
     import_droppy = args["--import-droppy"]
     necessary_opts = exists or import_droppy or path
     if not necessary_opts:
@@ -117,7 +119,8 @@ def _confdir(args):
                 raise ValueError("Config path is not a directory")
             # Accidentally pointed to the db.toml, use parent
             confdir = confdir.parent
-        config.conffile = confdir / config.conffile.name
+        os.environ["CISTA_HOME"] = confdir.as_posix()
+    config.init_confdir()  # Uses environ if available
 
 
 def _user(args):

cista/app.py

@@ -3,6 +3,7 @@ import datetime
 import mimetypes
 import threading
 from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import cpu_count
 from pathlib import Path, PurePath, PurePosixPath
 from stat import S_IFDIR, S_IFREG
 from urllib.parse import unquote
@@ -14,6 +15,7 @@ from blake3 import blake3
 from sanic import Blueprint, Sanic, empty, raw, redirect
 from sanic.exceptions import Forbidden, NotFound
 from sanic.log import logger
+from setproctitle import setproctitle
 from stream_zip import ZIP_AUTO, stream_zip
 
 from cista import auth, config, preview, session, watching
@@ -30,11 +32,16 @@ app.blueprint(bp)
 app.exception(Exception)(handle_sanic_exception)
 
 
+setproctitle("cista-main")
+
+
 @app.before_server_start
 async def main_start(app, loop):
     config.load_config()
+    setproctitle(f"cista {config.config.path.name}")
+    workers = max(2, min(8, cpu_count()))
     app.ctx.threadexec = ThreadPoolExecutor(
-        max_workers=8, thread_name_prefix="cista-ioworker"
+        max_workers=workers, thread_name_prefix="cista-ioworker"
     )
     await watching.start(app, loop)
 

cista/config.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import os
 import secrets
 import sys
+from contextlib import suppress
 from functools import wraps
 from hashlib import sha256
 from pathlib import Path, PurePath
@@ -33,7 +35,23 @@ class Link(msgspec.Struct, omit_defaults=True):
 
 
 config = None
-conffile = Path.home() / ".local/share/cista/db.toml"
+conffile = None
+
+
+def init_confdir():
+    if p := os.environ.get("CISTA_HOME"):
+        home = Path(p)
+    else:
+        xdg = os.environ.get("XDG_CONFIG_HOME")
+        home = (
+            Path(xdg).expanduser() / "cista" if xdg else Path.home() / ".config/cista"
+        )
+    if not home.is_dir():
+        home.mkdir(parents=True, exist_ok=True)
+        home.chmod(0o700)
+
+    global conffile
+    conffile = home / "db.toml"
 
 
 def derived_secret(*params, len=8) -> bytes:
@@ -61,8 +79,8 @@ def dec_hook(typ, obj):
 
 def config_update(modify):
     global config
-    if not conffile.exists():
-        conffile.parent.mkdir(parents=True, exist_ok=True)
+    if conffile is None:
+        init_confdir()
     tmpname = conffile.with_suffix(".tmp")
     try:
         f = tmpname.open("xb")
@@ -76,10 +94,6 @@ def config_update(modify):
             old = conffile.read_bytes()
             c = msgspec.toml.decode(old, type=Config, dec_hook=dec_hook)
         except FileNotFoundError:
-            # No existing config file, make sure we have a folder...
-            confdir = conffile.parent
-            confdir.mkdir(parents=True, exist_ok=True)
-            confdir.chmod(0o700)
             old = b""
             c = None
         c = modify(c)
@@ -92,7 +106,9 @@ def config_update(modify):
         f.write(new)
         f.close()
         if sys.platform == "win32":
-            conffile.unlink()  # Windows doesn't support atomic replace
+            # Windows doesn't support atomic replace
+            with suppress(FileNotFoundError):
+                conffile.unlink()
         tmpname.rename(conffile)  # Atomic replace
     except:
         f.close()
@@ -120,6 +136,8 @@ def modifies_config(modify):
 
 def load_config():
     global config
+    if conffile is None:
+        init_confdir()
     config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook)
 
 

cista/preview.py

@@ -1,4 +1,5 @@
 import asyncio
+import gc
 import io
 import mimetypes
 import urllib.parse
@@ -17,6 +18,8 @@ from sanic.log import logger
 from cista import config
 from cista.util.filename import sanitize
 
+DISPLAYMATRIX = av.stream.SideData.DISPLAYMATRIX
+
 bp = Blueprint("preview", url_prefix="/preview")
 
 
@@ -96,19 +99,19 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
 def process_video(path, *, maxsize, quality):
     with av.open(str(path)) as container:
         stream = container.streams.video[0]
-        rotation = (
-            stream.side_data
-            and stream.side_data.get(av.stream.SideData.DISPLAYMATRIX)
-            or 0
-        )
         stream.codec_context.skip_frame = "NONKEY"
+        rot = stream.side_data and stream.side_data.get(DISPLAYMATRIX) or 0
         container.seek(container.duration // 8)
-        frame = next(container.decode(stream))
-        img = frame.to_image()
+        img = next(container.decode(stream)).to_image()
+        del stream
 
     img.thumbnail((maxsize, maxsize))
     imgdata = io.BytesIO()
-    if rotation:
-        img = img.rotate(rotation, expand=True)
+    if rot:
+        img = img.rotate(rot, expand=True)
     img.save(imgdata, format="webp", quality=quality, method=4)
-    return imgdata.getvalue()
+    del img
+    ret = imgdata.getvalue()
+    del imgdata
+    gc.collect()
+    return ret

cista/serve.py

@@ -26,7 +26,6 @@ def run(*, dev=False):
         motd=False,
         dev=dev,
         auto_reload=dev,
-        reload_dir={confdir},
         access_log=True,
     )  # type: ignore
     if dev:

frontend/.npmrc

@@ -0,0 +1,2 @@
+audit=false
+fund=false

frontend/package.json

@@ -12,6 +12,9 @@
     "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
     "format": "prettier --write src/"
   },
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "dependencies": {
     "@imengyu/vue3-context-menu": "^1.3.3",
     "@vueuse/core": "^10.4.1",
@@ -21,7 +24,6 @@
     "pinia": "^2.1.6",
     "pinia-plugin-persistedstate": "^3.2.0",
     "unplugin-vue-components": "^0.25.2",
-    "vite-plugin-rewrite-all": "^1.0.1",
     "vite-svg-loader": "^4.0.0",
     "vue": "^3.3.4",
     "vue-router": "^4.2.4"

frontend/vite.config.ts

@@ -4,7 +4,6 @@ import { defineConfig } from 'vite'
 import vue from '@vitejs/plugin-vue'
 
 // @ts-ignore
-import pluginRewriteAll from 'vite-plugin-rewrite-all'
 import svgLoader from 'vite-svg-loader'
 import Components from 'unplugin-vue-components/vite'
 
@@ -21,7 +20,6 @@ const dev_backend = {
 export default defineConfig({
   plugins: [
     vue(),
-    pluginRewriteAll(),
     svgLoader(),          // import svg files
     Components(),         // auto import components
   ],

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "pyjwt",
     "pymupdf",
     "sanic",
+    "setproctitle",
     "stream-zip",
     "tomli_w",
 ]

scripts/build-frontend.py

@@ -1,5 +1,8 @@
 # noqa: INP001
+import os
+import shutil
 import subprocess
+from sys import stderr
 
 from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 
@@ -7,6 +10,21 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 class CustomBuildHook(BuildHookInterface):
     def initialize(self, version, build_data):
         super().initialize(version, build_data)
-        print("Building Cista frontend...")
-        subprocess.run("npm install --prefix frontend".split(" "), check=True)  # noqa: S603
-        subprocess.run("npm run build --prefix frontend".split(" "), check=True)  # noqa: S603
+        # A hack to stop building twice on run
+        if not build_data.get("force_include"):
+            return
+        stderr.write(">>> Building Cista frontend\n")
+        npm = shutil.which("npm")
+        if npm is None:
+            raise RuntimeError(
+                "NodeJS `npm` is required for building Cista but it was not found"
+            )
+        # npm --prefix doesn't work on Windows, so we chdir instead
+        os.chdir("frontend")
+        try:
+            stderr.write("### npm install\n")
+            subprocess.run([npm, "install"], check=True)  # noqa: S603
+            stderr.write("\n### npm run build\n")
+            subprocess.run([npm, "run", "build"], check=True)  # noqa: S603
+        finally:
+            os.chdir("..")