From e2a9a6903c42233c9fd80fa0f8caf32793b3af02 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 03:46:06 -0800 Subject: [PATCH 01/34] Memtrace --- cista/app.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cista/app.py b/cista/app.py index e646fe4..7a1fffa 100644 --- a/cista/app.py +++ b/cista/app.py @@ -2,6 +2,7 @@ import asyncio import datetime import mimetypes import threading +import tracemalloc from concurrent.futures import ThreadPoolExecutor from pathlib import Path, PurePath, PurePosixPath from stat import S_IFDIR, S_IFREG @@ -9,6 +10,7 @@ from urllib.parse import unquote from wsgiref.handlers import format_date_time import brotli +import objgraph import sanic.helpers from blake3 import blake3 from sanic import Blueprint, Sanic, empty, raw, redirect @@ -32,6 +34,7 @@ app.exception(Exception)(handle_sanic_exception) @app.before_server_start async def main_start(app, loop): + tracemalloc.start() config.load_config() app.ctx.threadexec = ThreadPoolExecutor( max_workers=8, thread_name_prefix="cista-ioworker" @@ -39,6 +42,17 @@ async def main_start(app, loop): await watching.start(app, loop) +@app.add_task +async def mem_task(): + while True: + await asyncio.sleep(10) + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics("lineno") + for stat in top_stats[:10]: + print(stat) + objgraph.show_growth(limit=10) + + @app.after_server_stop async def main_stop(app, loop): quit.set() -- 2.45.2 From c3d6aecffd9d2231f310d0973ce293d5f2e99271 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 04:00:20 -0800 Subject: [PATCH 02/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cista/preview.py b/cista/preview.py index e456599..dbb2893 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -1,4 +1,5 @@ import asyncio +import gc import io import mimetypes import urllib.parse @@ -105,10 +106,13 @@ def process_video(path, *, maxsize, quality): container.seek(container.duration // 8) frame = next(container.decode(stream)) img = frame.to_image() + del frame, stream img.thumbnail((maxsize, maxsize)) imgdata = io.BytesIO() if rotation: img = img.rotate(rotation, expand=True) img.save(imgdata, format="webp", quality=quality, method=4) - return imgdata.getvalue() + ret = imgdata.getvalue() + gc.collect() + return ret -- 2.45.2 From 27422ae1e2cefccf34549e030eb19e2971be7939 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 04:01:51 -0800 Subject: [PATCH 03/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cista/preview.py b/cista/preview.py index dbb2893..fed9c8e 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -108,11 +108,13 @@ def process_video(path, *, maxsize, quality): img = frame.to_image() del frame, stream - img.thumbnail((maxsize, maxsize)) - imgdata = io.BytesIO() - if rotation: - img = img.rotate(rotation, expand=True) - img.save(imgdata, format="webp", quality=quality, method=4) + img.thumbnail((maxsize, maxsize)) + imgdata = io.BytesIO() + if rotation: + img = img.rotate(rotation, expand=True) + img.save(imgdata, format="webp", quality=quality, method=4) + del img + ret = imgdata.getvalue() gc.collect() return ret -- 2.45.2 From 71eb252b8d9ba5cd607b368e3f55a2d2b6bdbf3e Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:13:57 +0000 Subject: [PATCH 04/34] Restrict the number of workers. --- cista/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cista/app.py b/cista/app.py index 7a1fffa..bdacb03 100644 --- a/cista/app.py +++ b/cista/app.py @@ -37,7 +37,7 @@ async def main_start(app, loop): tracemalloc.start() config.load_config() app.ctx.threadexec = ThreadPoolExecutor( - max_workers=8, thread_name_prefix="cista-ioworker" + max_workers=3, thread_name_prefix="cista-ioworker" ) await watching.start(app, loop) -- 2.45.2 From 02c5e484b56f62eb85ab26613bc1d4d4b350eda1 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:22:03 +0000 Subject: [PATCH 05/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/cista/preview.py b/cista/preview.py index fed9c8e..cb541e3 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -1,5 +1,4 @@ import asyncio -import gc import io import mimetypes import urllib.parse @@ -10,6 +9,7 @@ from wsgiref.handlers import format_date_time import av import av.datasets import fitz # PyMuPDF +from av.streams import SideData from PIL import Image from sanic import Blueprint, empty, raw from sanic.exceptions import NotFound @@ -97,24 +97,17 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0): def process_video(path, *, maxsize, quality): with av.open(str(path)) as container: stream = container.streams.video[0] - rotation = ( - stream.side_data - and stream.side_data.get(av.stream.SideData.DISPLAYMATRIX) - or 0 - ) + rot = stream.side_data and stream.side_data.get(SideData.DISPLAYMATRIX) or 0 stream.codec_context.skip_frame = "NONKEY" container.seek(container.duration // 8) - frame = next(container.decode(stream)) - img = frame.to_image() - del frame, stream + img = next(container.decode(stream)).to_image() + del stream - img.thumbnail((maxsize, maxsize)) - imgdata = io.BytesIO() - if rotation: - img = img.rotate(rotation, expand=True) - img.save(imgdata, format="webp", quality=quality, method=4) - del img + img.thumbnail((maxsize, maxsize)) + imgdata = io.BytesIO() + if rot: + img = img.rotate(rot, expand=True) + img.save(imgdata, format="webp", quality=quality, method=4) + del img - ret = imgdata.getvalue() - gc.collect() - return ret + return imgdata.getvalue() -- 2.45.2 From 8437c1f60e8da0839ee93a59650b4d1987b8ef2e Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:23:24 +0000 Subject: [PATCH 06/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cista/preview.py b/cista/preview.py index cb541e3..0a82d2e 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -1,4 +1,5 @@ import asyncio +import gc import io import mimetypes import urllib.parse @@ -109,5 +110,7 @@ def process_video(path, *, maxsize, quality): img = img.rotate(rot, expand=True) img.save(imgdata, format="webp", quality=quality, method=4) del img - - return imgdata.getvalue() + ret = imgdata.getvalue() + del imgdata + gc.collect() + return ret -- 2.45.2 From fdbf9b26108f813be8ac7f18a033be35aa3211c8 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:26:16 +0000 Subject: [PATCH 07/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cista/preview.py b/cista/preview.py index 0a82d2e..bf9eb2d 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -98,8 +98,9 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0): def process_video(path, *, maxsize, quality): with av.open(str(path)) as container: stream = container.streams.video[0] - rot = stream.side_data and stream.side_data.get(SideData.DISPLAYMATRIX) or 0 stream.codec_context.skip_frame = "NONKEY" + stream.codec_context.threads = 1 + rot = stream.side_data and stream.side_data.get(SideData.DISPLAYMATRIX) or 0 container.seek(container.duration // 8) img = next(container.decode(stream)).to_image() del stream -- 2.45.2 From 728cd47dcaca138a193fda2d2b223756dc24032a Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:27:38 +0000 Subject: [PATCH 08/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cista/preview.py b/cista/preview.py index bf9eb2d..2b20a97 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -10,7 +10,6 @@ from wsgiref.handlers import format_date_time import av import av.datasets import fitz # PyMuPDF -from av.streams import SideData from PIL import Image from sanic import Blueprint, empty, raw from sanic.exceptions import NotFound @@ -19,6 +18,8 @@ from sanic.log import logger from cista import config from cista.util.filename import sanitize +DISPLAYMATRIX = av.streams.SideData.DISPLAYMATRIX + bp = Blueprint("preview", url_prefix="/preview") @@ -100,7 +101,7 @@ def process_video(path, *, maxsize, quality): stream = container.streams.video[0] stream.codec_context.skip_frame = "NONKEY" stream.codec_context.threads = 1 - rot = stream.side_data and stream.side_data.get(SideData.DISPLAYMATRIX) or 0 + rot = stream.side_data and stream.side_data.get(DISPLAYMATRIX) or 0 container.seek(container.duration // 8) img = next(container.decode(stream)).to_image() del stream -- 2.45.2 From 53f717e93a0c2fed9462a3e3771e862280c9115b Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:27:54 +0000 Subject: [PATCH 09/34] Attempt to reduce leak of ffmpeg previews --- cista/preview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cista/preview.py b/cista/preview.py index 2b20a97..23580bf 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -18,7 +18,7 @@ from sanic.log import logger from cista import config from cista.util.filename import sanitize -DISPLAYMATRIX = av.streams.SideData.DISPLAYMATRIX +DISPLAYMATRIX = av.stream.SideData.DISPLAYMATRIX bp = Blueprint("preview", url_prefix="/preview") -- 2.45.2 From 3c79f06d2c3fb253ba4f0002db67d3b43089a124 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:30:23 +0000 Subject: [PATCH 10/34] Don't restart on config changes. --- cista/serve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cista/serve.py b/cista/serve.py index 24c9638..280ba31 100644 --- a/cista/serve.py +++ b/cista/serve.py @@ -26,7 +26,6 @@ def run(*, dev=False): motd=False, dev=dev, auto_reload=dev, - reload_dir={confdir}, access_log=True, ) # type: ignore if dev: -- 2.45.2 From 888bc9726463771407fe8ae1d1c59be8d62104b6 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:35:56 +0000 Subject: [PATCH 11/34] Don't restart on config changes. --- cista/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cista/app.py b/cista/app.py index bdacb03..7cad9a4 100644 --- a/cista/app.py +++ b/cista/app.py @@ -36,6 +36,7 @@ app.exception(Exception)(handle_sanic_exception) async def main_start(app, loop): tracemalloc.start() config.load_config() + print(config.config, config.config.public) app.ctx.threadexec = ThreadPoolExecutor( max_workers=3, thread_name_prefix="cista-ioworker" ) -- 2.45.2 From 37e06dd412df93a20d8d0c012d448b409f2f379f Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:39:02 +0000 Subject: [PATCH 12/34] Don't restart on config changes. --- cista/droppy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cista/droppy.py b/cista/droppy.py index 3271611..cb77d80 100644 --- a/cista/droppy.py +++ b/cista/droppy.py @@ -5,6 +5,7 @@ import msgspec def readconf() -> dict: p = Path.home() / ".droppy/config" # Hardcoded in Droppy + print("Reading Droppy config", p) cf = msgspec.json.decode((p / "config.json").read_bytes()) db = msgspec.json.decode((p / "db.json").read_bytes()) cf["path"] = p.parent / "files" -- 2.45.2 From d2ad269df89fbc51b92c50ef38d615d0bc2a01e3 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:48:18 +0000 Subject: [PATCH 13/34] Fix confdir passing to workers --- cista/__main__.py | 2 ++ cista/app.py | 1 - cista/config.py | 3 ++- cista/droppy.py | 1 - 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cista/__main__.py b/cista/__main__.py index 407c60f..f50bfb8 100644 --- a/cista/__main__.py +++ b/cista/__main__.py @@ -1,3 +1,4 @@ +import os import sys from pathlib import Path @@ -117,6 +118,7 @@ def _confdir(args): raise ValueError("Config path is not a directory") # Accidentally pointed to the db.toml, use parent confdir = confdir.parent + os.environ["CISTA_HOME"] = confdir.as_posix() config.conffile = confdir / config.conffile.name diff --git a/cista/app.py b/cista/app.py index 7cad9a4..bdacb03 100644 --- a/cista/app.py +++ b/cista/app.py @@ -36,7 +36,6 @@ app.exception(Exception)(handle_sanic_exception) async def main_start(app, loop): tracemalloc.start() config.load_config() - print(config.config, config.config.public) app.ctx.threadexec = ThreadPoolExecutor( max_workers=3, thread_name_prefix="cista-ioworker" ) diff --git a/cista/config.py b/cista/config.py index 3ecf9af..823451d 100644 --- a/cista/config.py +++ b/cista/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import secrets import sys from functools import wraps @@ -33,7 +34,7 @@ class Link(msgspec.Struct, omit_defaults=True): config = None -conffile = Path.home() / ".local/share/cista/db.toml" +conffile = os.environ.get("CISTA_HOME") or Path.home() / ".local/share/cista/db.toml" def derived_secret(*params, len=8) -> bytes: diff --git a/cista/droppy.py b/cista/droppy.py index cb77d80..3271611 100644 --- a/cista/droppy.py +++ b/cista/droppy.py @@ -5,7 +5,6 @@ import msgspec def readconf() -> dict: p = Path.home() / ".droppy/config" # Hardcoded in Droppy - print("Reading Droppy config", p) cf = msgspec.json.decode((p / "config.json").read_bytes()) db = msgspec.json.decode((p / "db.json").read_bytes()) cf["path"] = p.parent / "files" -- 2.45.2 From a267a679473b4f6504f9e4be6ad4f8597f8c5bdb Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 12:52:42 +0000 Subject: [PATCH 14/34] Fix confdir passing to workers --- cista/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cista/config.py b/cista/config.py index 823451d..bb906bb 100644 --- a/cista/config.py +++ b/cista/config.py @@ -34,7 +34,7 @@ class Link(msgspec.Struct, omit_defaults=True): config = None -conffile = os.environ.get("CISTA_HOME") or Path.home() / ".local/share/cista/db.toml" +conffile = None def derived_secret(*params, len=8) -> bytes: @@ -120,7 +120,10 @@ def modifies_config(modify): def load_config(): - global config + global config, conffile + conffile = ( + Path(os.environ.get("CISTA_HOME")) or Path.home() / ".local/share/cista/db.toml" + ) config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook) -- 2.45.2 From 13e1a19c5ba9c104cb15583a901804022e68679c Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:05:55 +0000 Subject: [PATCH 15/34] Fix confdir passing to workers --- cista/config.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/cista/config.py b/cista/config.py index bb906bb..83f6abf 100644 --- a/cista/config.py +++ b/cista/config.py @@ -7,7 +7,7 @@ from functools import wraps from hashlib import sha256 from pathlib import Path, PurePath from time import time - +from contextlib import suppress import msgspec @@ -37,6 +37,22 @@ config = None conffile = None +def init_confdir(): + if p := os.environ.get("CISTA_HOME"): + home = Path(p) + else: + xdg = os.environ.get("XDG_CONFIG_HOME") + home = ( + Path(xdg).expanduser() / "cista" if xdg else Path.home() / ".config/cista" + ) + if not home.exists() + home.mkdir(parents=True, exist_ok=True) + home.chmod(0o700) + + global conffile + conffile = home / "db.toml" + + def derived_secret(*params, len=8) -> bytes: """Used to derive secret keys from the main secret""" # Each part is made the same length by hashing first @@ -62,8 +78,8 @@ def dec_hook(typ, obj): def config_update(modify): global config - if not conffile.exists(): - conffile.parent.mkdir(parents=True, exist_ok=True) + if conffile is None: + init_confdir() tmpname = conffile.with_suffix(".tmp") try: f = tmpname.open("xb") @@ -77,10 +93,6 @@ def config_update(modify): old = conffile.read_bytes() c = msgspec.toml.decode(old, type=Config, dec_hook=dec_hook) except FileNotFoundError: - # No existing config file, make sure we have a folder... - confdir = conffile.parent - confdir.mkdir(parents=True, exist_ok=True) - confdir.chmod(0o700) old = b"" c = None c = modify(c) @@ -93,7 +105,9 @@ def config_update(modify): f.write(new) f.close() if sys.platform == "win32": - conffile.unlink() # Windows doesn't support atomic replace + # Windows doesn't support atomic replace + with suppress(FileNotFoundError): + conffile.unlink() tmpname.rename(conffile) # Atomic replace except: f.close() @@ -121,9 +135,8 @@ def modifies_config(modify): def load_config(): global config, conffile - conffile = ( - Path(os.environ.get("CISTA_HOME")) or Path.home() / ".local/share/cista/db.toml" - ) + if conffile is None: + init_confdir() config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook) -- 2.45.2 From ba87abad46824abcbba27c202cd79d2228404bcb Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:06:43 +0000 Subject: [PATCH 16/34] Fix confdir passing to workers --- cista/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cista/__main__.py b/cista/__main__.py index f50bfb8..f541de3 100644 --- a/cista/__main__.py +++ b/cista/__main__.py @@ -50,6 +50,7 @@ def main(): def _main(): args = docopt(doc) + config.init_confdir() if args["--user"]: return _user(args) listen = args["-l"] -- 2.45.2 From a1029d2c551c0e9cf5bb7718b3f2a1bae48e1c7e Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:08:20 +0000 Subject: [PATCH 17/34] Fix confdir passing to workers --- cista/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cista/__main__.py b/cista/__main__.py index f541de3..7912ff5 100644 --- a/cista/__main__.py +++ b/cista/__main__.py @@ -50,7 +50,6 @@ def main(): def _main(): args = docopt(doc) - config.init_confdir() if args["--user"]: return _user(args) listen = args["-l"] @@ -120,7 +119,7 @@ def _confdir(args): # Accidentally pointed to the db.toml, use parent confdir = confdir.parent os.environ["CISTA_HOME"] = confdir.as_posix() - config.conffile = confdir / config.conffile.name + config.init_confdir() # Uses environ if available def _user(args): -- 2.45.2 From 8978e91a68cf959f9eb430f2962bc38754026f01 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:09:16 +0000 Subject: [PATCH 18/34] Fix confdir passing to workers --- cista/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cista/config.py b/cista/config.py index 83f6abf..6f88213 100644 --- a/cista/config.py +++ b/cista/config.py @@ -3,11 +3,12 @@ from __future__ import annotations import os import secrets import sys +from contextlib import suppress from functools import wraps from hashlib import sha256 from pathlib import Path, PurePath from time import time -from contextlib import suppress + import msgspec @@ -45,7 +46,7 @@ def init_confdir(): home = ( Path(xdg).expanduser() / "cista" if xdg else Path.home() / ".config/cista" ) - if not home.exists() + if not home.is_dir(): home.mkdir(parents=True, exist_ok=True) home.chmod(0o700) -- 2.45.2 From e93b94a05c96325ca207e95a1f5bbd243fcafc62 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:10:51 +0000 Subject: [PATCH 19/34] Fix confdir passing to workers --- cista/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cista/__main__.py b/cista/__main__.py index 7912ff5..3234de6 100644 --- a/cista/__main__.py +++ b/cista/__main__.py @@ -62,6 +62,7 @@ def _main(): path = None _confdir(args) exists = config.conffile.exists() + print(config.conffile, exists) import_droppy = args["--import-droppy"] necessary_opts = exists or import_droppy or path if not necessary_opts: -- 2.45.2 From 6e236481a16ad6d5dd726d10979090ca4c20c679 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:13:57 +0000 Subject: [PATCH 20/34] Fix confdir passing to workers --- cista/preview.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cista/preview.py b/cista/preview.py index 23580bf..a595057 100644 --- a/cista/preview.py +++ b/cista/preview.py @@ -100,7 +100,6 @@ def process_video(path, *, maxsize, quality): with av.open(str(path)) as container: stream = container.streams.video[0] stream.codec_context.skip_frame = "NONKEY" - stream.codec_context.threads = 1 rot = stream.side_data and stream.side_data.get(DISPLAYMATRIX) or 0 container.seek(container.duration // 8) img = next(container.decode(stream)).to_image() -- 2.45.2 From 0be61cff2b7617b21362bb084019967e856e5e7b Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:18:30 +0000 Subject: [PATCH 21/34] Fix confdir passing to workers --- cista/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cista/__main__.py b/cista/__main__.py index 3234de6..5822d15 100644 --- a/cista/__main__.py +++ b/cista/__main__.py @@ -119,7 +119,7 @@ def _confdir(args): raise ValueError("Config path is not a directory") # Accidentally pointed to the db.toml, use parent confdir = confdir.parent - os.environ["CISTA_HOME"] = confdir.as_posix() + os.environ["CISTA_HOME"] = confdir.as_posix() config.init_confdir() # Uses environ if available -- 2.45.2 From 85693e99d04740eec8b5eae14f9477ce20e6bb47 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:35:54 +0000 Subject: [PATCH 22/34] Fix build --- scripts/build-frontend.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/build-frontend.py b/scripts/build-frontend.py index 10c1756..4bd0d43 100644 --- a/scripts/build-frontend.py +++ b/scripts/build-frontend.py @@ -1,4 +1,5 @@ # noqa: INP001 +import shutil import subprocess from hatchling.builders.hooks.plugin.interface import BuildHookInterface @@ -7,6 +8,15 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface class CustomBuildHook(BuildHookInterface): def initialize(self, version, build_data): super().initialize(version, build_data) - print("Building Cista frontend...") - subprocess.run("npm install --prefix frontend".split(" "), check=True) # noqa: S603 - subprocess.run("npm run build --prefix frontend".split(" "), check=True) # noqa: S603 + # A hack to stop building twice on run + if not build_data.get("force_include"): + return + print("Building Cista frontend...", version, build_data) + npm = shutil.which("npm") + if npm is None: + raise RuntimeError( + "NodeJS `npm` is required for building Cista but it was not found" + ) + frontend = "frontend" + subprocess.run([npm, "install", "--prefix", frontend], check=True) # noqa: S603 + subprocess.run([npm, "run", "build", "--prefix", frontend], check=True) # noqa: S603 -- 2.45.2 From 226ac67d26724a117acc928f28c267d0ac0dc278 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:40:17 +0000 Subject: [PATCH 23/34] Fix build --- scripts/build-frontend.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/build-frontend.py b/scripts/build-frontend.py index 4bd0d43..769f73a 100644 --- a/scripts/build-frontend.py +++ b/scripts/build-frontend.py @@ -1,4 +1,5 @@ # noqa: INP001 +import os import shutil import subprocess @@ -17,6 +18,10 @@ class CustomBuildHook(BuildHookInterface): raise RuntimeError( "NodeJS `npm` is required for building Cista but it was not found" ) - frontend = "frontend" - subprocess.run([npm, "install", "--prefix", frontend], check=True) # noqa: S603 - subprocess.run([npm, "run", "build", "--prefix", frontend], check=True) # noqa: S603 + # npm --prefix doesn't work on Windows, so we chdir instead + os.chdir("frontend") + try: + subprocess.run([npm, "install"], check=True) # noqa: S603 + subprocess.run([npm, "run", "build"], check=True) # noqa: S603 + finally: + os.chdir("..") -- 2.45.2 From 7ff57924ae3cdf54f6a32444dbc12a6514838565 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 13:52:56 +0000 Subject: [PATCH 24/34] Fix build --- scripts/build-frontend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/build-frontend.py b/scripts/build-frontend.py index 769f73a..b13c575 100644 --- a/scripts/build-frontend.py +++ b/scripts/build-frontend.py @@ -2,6 +2,7 @@ import os import shutil import subprocess +from sys import stderr from hatchling.builders.hooks.plugin.interface import BuildHookInterface @@ -12,7 +13,7 @@ class CustomBuildHook(BuildHookInterface): # A hack to stop building twice on run if not build_data.get("force_include"): return - print("Building Cista frontend...", version, build_data) + stderr.write("🏗️ Building Cista frontend\n") npm = shutil.which("npm") if npm is None: raise RuntimeError( @@ -21,7 +22,9 @@ class CustomBuildHook(BuildHookInterface): # npm --prefix doesn't work on Windows, so we chdir instead os.chdir("frontend") try: + stderr.write(" 》npm install\n") subprocess.run([npm, "install"], check=True) # noqa: S603 + stderr.write("\n 》npm run build\n") subprocess.run([npm, "run", "build"], check=True) # noqa: S603 finally: os.chdir("..") -- 2.45.2 From ec2cad5e67ffbfba63c8872d35b9a93a08253a1d Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 14:06:21 +0000 Subject: [PATCH 25/34] Fix build --- frontend/package.json | 3 +++ scripts/build-frontend.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 4686e02..67daf9d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,9 @@ "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore", "format": "prettier --write src/" }, + "engines": { + "node": ">=18.0.0" + }, "dependencies": { "@imengyu/vue3-context-menu": "^1.3.3", "@vueuse/core": "^10.4.1", diff --git a/scripts/build-frontend.py b/scripts/build-frontend.py index b13c575..7f94070 100644 --- a/scripts/build-frontend.py +++ b/scripts/build-frontend.py @@ -13,7 +13,7 @@ class CustomBuildHook(BuildHookInterface): # A hack to stop building twice on run if not build_data.get("force_include"): return - stderr.write("🏗️ Building Cista frontend\n") + stderr.write(">>> Building Cista frontend\n") npm = shutil.which("npm") if npm is None: raise RuntimeError( @@ -22,9 +22,9 @@ class CustomBuildHook(BuildHookInterface): # npm --prefix doesn't work on Windows, so we chdir instead os.chdir("frontend") try: - stderr.write(" 》npm install\n") + stderr.write("### npm install\n") subprocess.run([npm, "install"], check=True) # noqa: S603 - stderr.write("\n 》npm run build\n") + stderr.write("\n### npm run build\n") subprocess.run([npm, "run", "build"], check=True) # noqa: S603 finally: os.chdir("..") -- 2.45.2 From be744cf8e10982c324cf1bea01a5048f5f29a553 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 14:10:56 +0000 Subject: [PATCH 26/34] Fix build --- frontend/.npmrc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 frontend/.npmrc diff --git a/frontend/.npmrc b/frontend/.npmrc new file mode 100644 index 0000000..e8127a5 --- /dev/null +++ b/frontend/.npmrc @@ -0,0 +1,3 @@ +audit=false +fund=false +package-lock=false -- 2.45.2 From 56bc9b7cc8e07bd7c007c1f19bbd335c5ff9b98f Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 14:13:35 +0000 Subject: [PATCH 27/34] Fix build --- frontend/package.json | 1 - frontend/vite.config.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 67daf9d..27894cd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -24,7 +24,6 @@ "pinia": "^2.1.6", "pinia-plugin-persistedstate": "^3.2.0", "unplugin-vue-components": "^0.25.2", - "vite-plugin-rewrite-all": "^1.0.1", "vite-svg-loader": "^4.0.0", "vue": "^3.3.4", "vue-router": "^4.2.4" diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 2de1a87..834d0fd 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -4,7 +4,6 @@ import { defineConfig } from 'vite' import vue from '@vitejs/plugin-vue' // @ts-ignore -import pluginRewriteAll from 'vite-plugin-rewrite-all' import svgLoader from 'vite-svg-loader' import Components from 'unplugin-vue-components/vite' @@ -21,7 +20,6 @@ const dev_backend = { export default defineConfig({ plugins: [ vue(), - pluginRewriteAll(), svgLoader(), // import svg files Components(), // auto import components ], -- 2.45.2 From f7e968666e7e8248d18ae9bcac765f472c8b6883 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 14:20:05 +0000 Subject: [PATCH 28/34] Remove memory trackign --- cista/app.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/cista/app.py b/cista/app.py index bdacb03..6c879b5 100644 --- a/cista/app.py +++ b/cista/app.py @@ -2,7 +2,6 @@ import asyncio import datetime import mimetypes import threading -import tracemalloc from concurrent.futures import ThreadPoolExecutor from pathlib import Path, PurePath, PurePosixPath from stat import S_IFDIR, S_IFREG @@ -10,7 +9,6 @@ from urllib.parse import unquote from wsgiref.handlers import format_date_time import brotli -import objgraph import sanic.helpers from blake3 import blake3 from sanic import Blueprint, Sanic, empty, raw, redirect @@ -34,7 +32,6 @@ app.exception(Exception)(handle_sanic_exception) @app.before_server_start async def main_start(app, loop): - tracemalloc.start() config.load_config() app.ctx.threadexec = ThreadPoolExecutor( max_workers=3, thread_name_prefix="cista-ioworker" @@ -42,17 +39,6 @@ async def main_start(app, loop): await watching.start(app, loop) -@app.add_task -async def mem_task(): - while True: - await asyncio.sleep(10) - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics("lineno") - for stat in top_stats[:10]: - print(stat) - objgraph.show_growth(limit=10) - - @app.after_server_stop async def main_stop(app, loop): quit.set() -- 2.45.2 From 16a801152651d214b28cf5ac1da8769aaff986d8 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 14:21:53 +0000 Subject: [PATCH 29/34] Fix build --- frontend/.npmrc | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/.npmrc b/frontend/.npmrc index e8127a5..09b35cd 100644 --- a/frontend/.npmrc +++ b/frontend/.npmrc @@ -1,3 +1,2 @@ audit=false fund=false -package-lock=false -- 2.45.2 From 058bc9f912afea83e974fb5f8d94307a1fb9e359 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 14:36:42 +0000 Subject: [PATCH 30/34] Adaptive worker thread count --- cista/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cista/app.py b/cista/app.py index 6c879b5..49e33cf 100644 --- a/cista/app.py +++ b/cista/app.py @@ -3,6 +3,7 @@ import datetime import mimetypes import threading from concurrent.futures import ThreadPoolExecutor +from multiprocessing import cpu_count from pathlib import Path, PurePath, PurePosixPath from stat import S_IFDIR, S_IFREG from urllib.parse import unquote @@ -33,8 +34,9 @@ app.exception(Exception)(handle_sanic_exception) @app.before_server_start async def main_start(app, loop): config.load_config() + N = max(2, min(8, cpu_count())) app.ctx.threadexec = ThreadPoolExecutor( - max_workers=3, thread_name_prefix="cista-ioworker" + max_workers=N, thread_name_prefix="cista-ioworker" ) await watching.start(app, loop) -- 2.45.2 From f0701b32f6432ac09f3ac2a7db6a589ad5f71090 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 15:12:53 +0000 Subject: [PATCH 31/34] Improve process naming (only on Linux). --- cista/app.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cista/app.py b/cista/app.py index 49e33cf..1dbfa96 100644 --- a/cista/app.py +++ b/cista/app.py @@ -8,6 +8,8 @@ from pathlib import Path, PurePath, PurePosixPath from stat import S_IFDIR, S_IFREG from urllib.parse import unquote from wsgiref.handlers import format_date_time +import multiprocessing +import sys import brotli import sanic.helpers @@ -31,9 +33,24 @@ app.blueprint(bp) app.exception(Exception)(handle_sanic_exception) +def procname(name): + if sys.platform == "linux": + from ctypes import cdll, byref, create_string_buffer + + newname = name.encode() + libc = cdll.LoadLibrary("libc.so.6") + buff = create_string_buffer(len(newname) + 1) + buff.value = newname + libc.prctl(15, byref(buff), 0, 0, 0) + + +procname("cista-main") + + @app.before_server_start async def main_start(app, loop): config.load_config() + procname(f"cista {config.config.path.name}") N = max(2, min(8, cpu_count())) app.ctx.threadexec = ThreadPoolExecutor( max_workers=N, thread_name_prefix="cista-ioworker" -- 2.45.2 From a65c03757d972f1f8895e94508dea125a24bf4f6 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 15:20:49 +0000 Subject: [PATCH 32/34] Portable process naming module --- cista/app.py | 17 +++-------------- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/cista/app.py b/cista/app.py index 1dbfa96..9d3755c 100644 --- a/cista/app.py +++ b/cista/app.py @@ -10,7 +10,7 @@ from urllib.parse import unquote from wsgiref.handlers import format_date_time import multiprocessing import sys - +from secproctitle import setproctitle import brotli import sanic.helpers from blake3 import blake3 @@ -33,24 +33,13 @@ app.blueprint(bp) app.exception(Exception)(handle_sanic_exception) -def procname(name): - if sys.platform == "linux": - from ctypes import cdll, byref, create_string_buffer - - newname = name.encode() - libc = cdll.LoadLibrary("libc.so.6") - buff = create_string_buffer(len(newname) + 1) - buff.value = newname - libc.prctl(15, byref(buff), 0, 0, 0) - - -procname("cista-main") +setproctitle("cista-main") @app.before_server_start async def main_start(app, loop): config.load_config() - procname(f"cista {config.config.path.name}") + setproctitle(f"cista {config.config.path.name}") N = max(2, min(8, cpu_count())) app.ctx.threadexec = ThreadPoolExecutor( max_workers=N, thread_name_prefix="cista-ioworker" diff --git a/pyproject.toml b/pyproject.toml index 77d412c..ea6e73f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "pyjwt", "pymupdf", "sanic", + "setproctitle", "stream-zip", "tomli_w", ] -- 2.45.2 From ba9379ce9cf1ea60302461b07b48a5ddbc7512b9 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 15:29:16 +0000 Subject: [PATCH 33/34] Fix typo --- cista/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cista/app.py b/cista/app.py index 9d3755c..ab3f47a 100644 --- a/cista/app.py +++ b/cista/app.py @@ -10,7 +10,7 @@ from urllib.parse import unquote from wsgiref.handlers import format_date_time import multiprocessing import sys -from secproctitle import setproctitle +from setproctitle import setproctitle import brotli import sanic.helpers from blake3 import blake3 -- 2.45.2 From 8e93c96bb3f71e571706b243d6e8eb27289546db Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 21 Nov 2023 15:30:48 +0000 Subject: [PATCH 34/34] Cleanup --- cista/app.py | 9 ++++----- cista/config.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cista/app.py b/cista/app.py index ab3f47a..a87cef1 100644 --- a/cista/app.py +++ b/cista/app.py @@ -8,15 +8,14 @@ from pathlib import Path, PurePath, PurePosixPath from stat import S_IFDIR, S_IFREG from urllib.parse import unquote from wsgiref.handlers import format_date_time -import multiprocessing -import sys -from setproctitle import setproctitle + import brotli import sanic.helpers from blake3 import blake3 from sanic import Blueprint, Sanic, empty, raw, redirect from sanic.exceptions import Forbidden, NotFound from sanic.log import logger +from setproctitle import setproctitle from stream_zip import ZIP_AUTO, stream_zip from cista import auth, config, preview, session, watching @@ -40,9 +39,9 @@ setproctitle("cista-main") async def main_start(app, loop): config.load_config() setproctitle(f"cista {config.config.path.name}") - N = max(2, min(8, cpu_count())) + workers = max(2, min(8, cpu_count())) app.ctx.threadexec = ThreadPoolExecutor( - max_workers=N, thread_name_prefix="cista-ioworker" + max_workers=workers, thread_name_prefix="cista-ioworker" ) await watching.start(app, loop) diff --git a/cista/config.py b/cista/config.py index 6f88213..03fbfdd 100644 --- a/cista/config.py +++ b/cista/config.py @@ -135,7 +135,7 @@ def modifies_config(modify): def load_config(): - global config, conffile + global config if conffile is None: init_confdir() config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook) -- 2.45.2