Attempt to reduce leak of ffmpeg previews

cista/__main__.py

@@ -1,4 +1,3 @@
-import os
 import sys
 from pathlib import Path
 
@@ -62,7 +61,6 @@ def _main():
         path = None
     _confdir(args)
     exists = config.conffile.exists()
-    print(config.conffile, exists)
     import_droppy = args["--import-droppy"]
     necessary_opts = exists or import_droppy or path
     if not necessary_opts:
@@ -119,8 +117,7 @@ def _confdir(args):
                 raise ValueError("Config path is not a directory")
             # Accidentally pointed to the db.toml, use parent
             confdir = confdir.parent
-        os.environ["CISTA_HOME"] = confdir.as_posix()
-    config.init_confdir()  # Uses environ if available
+        config.conffile = confdir / config.conffile.name
 
 
 def _user(args):

cista/app.py

@@ -2,20 +2,20 @@ import asyncio
 import datetime
 import mimetypes
 import threading
+import tracemalloc
 from concurrent.futures import ThreadPoolExecutor
-from multiprocessing import cpu_count
 from pathlib import Path, PurePath, PurePosixPath
 from stat import S_IFDIR, S_IFREG
 from urllib.parse import unquote
 from wsgiref.handlers import format_date_time
 
 import brotli
+import objgraph
 import sanic.helpers
 from blake3 import blake3
 from sanic import Blueprint, Sanic, empty, raw, redirect
 from sanic.exceptions import Forbidden, NotFound
 from sanic.log import logger
-from setproctitle import setproctitle
 from stream_zip import ZIP_AUTO, stream_zip
 
 from cista import auth, config, preview, session, watching
@@ -32,20 +32,27 @@ app.blueprint(bp)
 app.exception(Exception)(handle_sanic_exception)
 
 
-setproctitle("cista-main")
-
-
 @app.before_server_start
 async def main_start(app, loop):
+    tracemalloc.start()
     config.load_config()
-    setproctitle(f"cista {config.config.path.name}")
-    workers = max(2, min(8, cpu_count()))
     app.ctx.threadexec = ThreadPoolExecutor(
-        max_workers=workers, thread_name_prefix="cista-ioworker"
+        max_workers=3, thread_name_prefix="cista-ioworker"
     )
     await watching.start(app, loop)
 
 
+@app.add_task
+async def mem_task():
+    while True:
+        await asyncio.sleep(10)
+        snapshot = tracemalloc.take_snapshot()
+        top_stats = snapshot.statistics("lineno")
+        for stat in top_stats[:10]:
+            print(stat)
+        objgraph.show_growth(limit=10)
+
+
 @app.after_server_stop
 async def main_stop(app, loop):
     quit.set()

cista/config.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 
-import os
 import secrets
 import sys
-from contextlib import suppress
 from functools import wraps
 from hashlib import sha256
 from pathlib import Path, PurePath
@@ -35,23 +33,7 @@ class Link(msgspec.Struct, omit_defaults=True):
 
 
 config = None
-conffile = None
-
-
-def init_confdir():
-    if p := os.environ.get("CISTA_HOME"):
-        home = Path(p)
-    else:
-        xdg = os.environ.get("XDG_CONFIG_HOME")
-        home = (
-            Path(xdg).expanduser() / "cista" if xdg else Path.home() / ".config/cista"
-        )
-    if not home.is_dir():
-        home.mkdir(parents=True, exist_ok=True)
-        home.chmod(0o700)
-
-    global conffile
-    conffile = home / "db.toml"
+conffile = Path.home() / ".local/share/cista/db.toml"
 
 
 def derived_secret(*params, len=8) -> bytes:
@@ -79,8 +61,8 @@ def dec_hook(typ, obj):
 
 def config_update(modify):
     global config
-    if conffile is None:
-        init_confdir()
+    if not conffile.exists():
+        conffile.parent.mkdir(parents=True, exist_ok=True)
     tmpname = conffile.with_suffix(".tmp")
     try:
         f = tmpname.open("xb")
@@ -94,6 +76,10 @@ def config_update(modify):
             old = conffile.read_bytes()
             c = msgspec.toml.decode(old, type=Config, dec_hook=dec_hook)
         except FileNotFoundError:
+            # No existing config file, make sure we have a folder...
+            confdir = conffile.parent
+            confdir.mkdir(parents=True, exist_ok=True)
+            confdir.chmod(0o700)
             old = b""
             c = None
         c = modify(c)
@@ -106,9 +92,7 @@ def config_update(modify):
         f.write(new)
         f.close()
         if sys.platform == "win32":
-            # Windows doesn't support atomic replace
-            with suppress(FileNotFoundError):
-                conffile.unlink()
+            conffile.unlink()  # Windows doesn't support atomic replace
         tmpname.rename(conffile)  # Atomic replace
     except:
         f.close()
@@ -136,8 +120,6 @@ def modifies_config(modify):
 
 def load_config():
     global config
-    if conffile is None:
-        init_confdir()
     config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook)
 
 

cista/preview.py

@@ -10,6 +10,7 @@ from wsgiref.handlers import format_date_time
 import av
 import av.datasets
 import fitz  # PyMuPDF
+from av.streams import SideData
 from PIL import Image
 from sanic import Blueprint, empty, raw
 from sanic.exceptions import NotFound
@@ -18,8 +19,6 @@ from sanic.log import logger
 from cista import config
 from cista.util.filename import sanitize
 
-DISPLAYMATRIX = av.stream.SideData.DISPLAYMATRIX
-
 bp = Blueprint("preview", url_prefix="/preview")
 
 
@@ -100,7 +99,8 @@ def process_video(path, *, maxsize, quality):
     with av.open(str(path)) as container:
         stream = container.streams.video[0]
         stream.codec_context.skip_frame = "NONKEY"
-        rot = stream.side_data and stream.side_data.get(DISPLAYMATRIX) or 0
+        stream.codec_context.threads = 1
+        rot = stream.side_data and stream.side_data.get(SideData.DISPLAYMATRIX) or 0
         container.seek(container.duration // 8)
         img = next(container.decode(stream)).to_image()
         del stream

cista/serve.py

@@ -26,6 +26,7 @@ def run(*, dev=False):
         motd=False,
         dev=dev,
         auto_reload=dev,
+        reload_dir={confdir},
         access_log=True,
     )  # type: ignore
     if dev:

frontend/.npmrc

@@ -1,2 +0,0 @@
-audit=false
-fund=false

frontend/package.json

@@ -12,9 +12,6 @@
     "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
     "format": "prettier --write src/"
   },
-  "engines": {
-    "node": ">=18.0.0"
-  },
   "dependencies": {
     "@imengyu/vue3-context-menu": "^1.3.3",
     "@vueuse/core": "^10.4.1",
@@ -24,6 +21,7 @@
     "pinia": "^2.1.6",
     "pinia-plugin-persistedstate": "^3.2.0",
     "unplugin-vue-components": "^0.25.2",
+    "vite-plugin-rewrite-all": "^1.0.1",
     "vite-svg-loader": "^4.0.0",
     "vue": "^3.3.4",
     "vue-router": "^4.2.4"

frontend/src/App.vue

@@ -10,10 +10,6 @@
   <main>
     <RouterView :path="path.pathList" :query="path.query" />
   </main>
-  <footer>
-    <TransferBar :status=store.uprogress @cancel=store.cancelUploads class=upload />
-    <TransferBar :status=store.dprogress @cancel=store.cancelDownloads class=download />
-  </footer>
 </template>
 
 <script setup lang="ts">

frontend/src/components/DownloadButton.vue

@@ -1,5 +1,6 @@
 <template>
   <SvgButton name="download" data-tooltip="Download" @click="download" />
+  <TransferBar :status=progress @cancel=cancelDownloads />
 </template>
 
 <script setup lang="ts">
@@ -25,22 +26,22 @@ const status_init = {
   filepos: 0,
   status: 'idle',
 }
-store.dprogress = {...status_init}
+const progress = reactive({...status_init})
 setInterval(() => {
-  if (Date.now() - store.dprogress.tlast > 3000) {
+  if (Date.now() - progress.tlast > 3000) {
     // Reset
-    store.dprogress.statbytes = 0
-    store.dprogress.statdur = 1
+    progress.statbytes = 0
+    progress.statdur = 1
   } else {
     // Running average by decay
-    store.dprogress.statbytes *= .9
-    store.dprogress.statdur *= .9
+    progress.statbytes *= .9
+    progress.statdur *= .9
   }
 }, 100)
 const statReset = () => {
-  Object.assign(store.dprogress, status_init)
-  store.dprogress.t0 = Date.now()
-  store.dprogress.tlast = store.dprogress.t0 + 1
+  Object.assign(progress, status_init)
+  progress.t0 = Date.now()
+  progress.tlast = progress.t0 + 1
 }
 const cancelDownloads = () => {
   location.reload()  // FIXME
@@ -60,9 +61,9 @@ const filesystemdl = async (sel: SelectedItems, handle: FileSystemDirectoryHandl
   console.log('Downloading to filesystem', sel.recursive)
   for (const [rel, full, doc] of sel.recursive) {
     if (doc.dir) continue
-    store.dprogress.files.push(rel)
-    ++store.dprogress.filecount
-    store.dprogress.total += doc.size
+    progress.files.push(rel)
+    ++progress.filecount
+    progress.total += doc.size
   }
   for (const [rel, full, doc] of sel.recursive) {
     // Create any missing directories
@@ -72,7 +73,6 @@ const filesystemdl = async (sel: SelectedItems, handle: FileSystemDirectoryHandl
     }
     const r = rel.slice(hdir.length)
     for (const dir of r.split('/').slice(0, doc.dir ? undefined : -1)) {
-      if (!dir) continue
       hdir += `${dir}/`
       try {
         h = await h.getDirectoryHandle(dir.normalize('NFC'), { create: true })
@@ -101,22 +101,22 @@ const filesystemdl = async (sel: SelectedItems, handle: FileSystemDirectoryHandl
       throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`)
     }
     if (res.body) {
-      ++store.dprogress.fileidx
+      ++progress.fileidx
       const reader = res.body.getReader()
       await writable.truncate(0)
       store.error = "Direct download."
-      store.dprogress.tlast = Date.now()
+      progress.tlast = Date.now()
       while (true) {
         const { value, done } = await reader.read()
         if (done) break
         await writable.write(value)
         const now = Date.now()
         const size = value.byteLength
-        store.dprogress.xfer += size
-        store.dprogress.filepos += size
-        store.dprogress.statbytes += size
-        store.dprogress.statdur += now - store.dprogress.tlast
-        store.dprogress.tlast = now
+        progress.xfer += size
+        progress.filepos += size
+        progress.statbytes += size
+        progress.statdur += now - progress.tlast
+        progress.tlast = now
       }
     }
     await writable.close()

frontend/src/components/FileExplorer.vue

@@ -115,7 +115,6 @@ const rename = (doc: Doc, newName: string) => {
 }
 defineExpose({
   newFolder() {
-    console.log("New folder")
     const now = Math.floor(Date.now() / 1000)
     editing.value = new Doc({
       loc: loc.value,
@@ -125,7 +124,6 @@ defineExpose({
       mtime: now,
       size: 0,
     })
-    store.cursor = editing.value.key
   },
   toggleSelectAll() {
     console.log('Select')

frontend/src/components/Gallery.vue

@@ -2,7 +2,7 @@
   <div v-if="props.documents.length || editing" class="gallery" ref="gallery">
     <GalleryFigure v-if="editing?.key === 'new'" :doc="editing" :key=editing.key :editing="{rename: mkdir, exit}" />
     <template v-for="(doc, index) in documents" :key=doc.key>
-      <GalleryFigure :doc=doc :editing="editing === doc ? {rename, exit} : null" @menu="contextMenu($event, doc)">
+      <GalleryFigure :doc=doc :editing="editing === doc ? {rename, exit} : null">
         <BreadCrumb v-if=showFolderBreadcrumb(index) :path="doc.loc ? doc.loc.split('/') : []" class="folder-change"/>
       </GalleryFigure>
     </template>
@@ -67,7 +67,6 @@ defineExpose({
       mtime: now,
       size: 0,
     })
-    store.cursor = editing.value.key
   },
   toggleSelectAll() {
     console.log('Select')

frontend/src/components/GalleryFigure.vue

@@ -9,7 +9,7 @@
       <slot></slot>
       <MediaPreview ref=m :doc="doc" tabindex=-1 quality="sz=512" class="figcontent" />
       <div class="titlespacer"></div>
-      <figcaption @click.prevent @contextmenu.prevent="$emit('menu', $event)">
+      <figcaption @click.prevent>
         <template v-if="editing">
           <FileRenameInput :doc=doc :rename=editing.rename :exit=editing.exit />
         </template>

frontend/src/components/HeaderMain.vue

@@ -8,7 +8,7 @@
     <SvgButton
       name="create-folder"
       data-tooltip="New folder"
-      @click="() => {  console.log('New', store.fileExplorer); store.fileExplorer!.newFolder(); console.log('Done')}"
+      @click="() => store.fileExplorer!.newFolder()"
     />
     <slot></slot>
     <div class="spacer smallgap"></div>

frontend/src/components/TransferBar.vue

@@ -57,12 +57,13 @@ const speeddisp = computed(() => speed.value ? speed.value.toFixed(speed.value <
   display: flex;
   flex-direction: column;
   color: var(--primary-color);
-  width: 100%;
+  position: fixed;
+  left: 0;
+  bottom: 0;
+  width: 100vw;
 }
 .statustext {
   display: flex;
-  align-items: center;
-  margin: 0 .5em;
   padding: 0.5rem 0;
 }
 span {
@@ -83,12 +84,4 @@ span {
 .position { min-width: 4em }
 .speed { min-width: 4em }
 
-.upload .statustext::before {
-  font-size: 1.5em;
-  content: '🔺'
-}
-.download .statustext::before {
-  font-size: 1.5em;
-  content: '🔻'
-}
 </style>

frontend/src/components/UploadButton.vue

@@ -1,12 +1,3 @@
-<template>
-  <template>
-    <input ref="fileInput" @change="uploadHandler" type="file" multiple>
-    <input ref="folderInput" @change="uploadHandler" type="file" webkitdirectory>
-  </template>
-  <SvgButton name="add-file" data-tooltip="Upload files" @click="fileInput.click()" />
-  <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderInput.click()" />
-</template>
-
 <script setup lang="ts">
 import { connect, uploadUrl } from '@/repositories/WS';
 import { useMainStore } from '@/stores/main'
@@ -117,50 +108,50 @@ const uprogress_init = {
   filepos: 0,
   status: 'idle',
 }
-store.uprogress = {...uprogress_init}
+const uprogress = reactive({...uprogress_init})
 setInterval(() => {
-  if (Date.now() - store.uprogress.tlast > 3000) {
+  if (Date.now() - uprogress.tlast > 3000) {
     // Reset
-    store.uprogress.statbytes = 0
-    store.uprogress.statdur = 1
+    uprogress.statbytes = 0
+    uprogress.statdur = 1
   } else {
     // Running average by decay
-    store.uprogress.statbytes *= .9
-    store.uprogress.statdur *= .9
+    uprogress.statbytes *= .9
+    uprogress.statdur *= .9
   }
 }, 100)
 const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => {
-  if (name !== store.uprogress.filename) return  // If stats have been reset
+  if (name !== uprogress.filename) return  // If stats have been reset
   const now = Date.now()
-  store.uprogress.xfer = store.uprogress.filestart + end
-  store.uprogress.filepos = end
-  store.uprogress.statbytes += end - start
-  store.uprogress.statdur += now - store.uprogress.tlast
-  store.uprogress.tlast = now
+  uprogress.xfer = uprogress.filestart + end
+  uprogress.filepos = end
+  uprogress.statbytes += end - start
+  uprogress.statdur += now - uprogress.tlast
+  uprogress.tlast = now
   // File finished?
   if (end === size) {
-    store.uprogress.filestart += size
+    uprogress.filestart += size
     statNextFile()
-    if (++store.uprogress.fileidx >= store.uprogress.filecount) statReset()
+    if (++uprogress.fileidx >= uprogress.filecount) statReset()
   }
 }
 const statNextFile = () => {
-  const f = store.uprogress.files.shift()
+  const f = uprogress.files.shift()
   if (!f) return statReset()
-  store.uprogress.filepos = 0
-  store.uprogress.filesize = f.file.size
-  store.uprogress.filename = f.cloudName
+  uprogress.filepos = 0
+  uprogress.filesize = f.file.size
+  uprogress.filename = f.cloudName
 }
 const statReset = () => {
-  Object.assign(store.uprogress, uprogress_init)
-  store.uprogress.t0 = Date.now()
-  store.uprogress.tlast = store.uprogress.t0 + 1
+  Object.assign(uprogress, uprogress_init)
+  uprogress.t0 = Date.now()
+  uprogress.tlast = uprogress.t0 + 1
 }
 const statsAdd = (f: CloudFile[]) => {
-  if (store.uprogress.files.length === 0) statReset()
-  store.uprogress.total += f.reduce((a, b) => a + b.file.size, 0)
-  store.uprogress.filecount += f.length
-  store.uprogress.files = [...store.uprogress.files, ...f]
+  if (uprogress.files.length === 0) statReset()
+  uprogress.total += f.reduce((a, b) => a + b.file.size, 0)
+  uprogress.filecount += f.length
+  uprogress.files = [...uprogress.files, ...f]
   statNextFile()
 }
 let upqueue = [] as CloudFile[]
@@ -190,7 +181,7 @@ const WSCreate = async () => await new Promise<WebSocket>(resolve => {
   // @ts-ignore
   ws.sendData = async (data: any) => {
     // Wait until the WS is ready to send another message
-    store.uprogress.status = "uploading"
+    uprogress.status = "uploading"
     await new Promise(resolve => {
       const t = setInterval(() => {
         if (ws.bufferedAmount > 1<<20) return
@@ -198,7 +189,7 @@ const WSCreate = async () => await new Promise<WebSocket>(resolve => {
         clearInterval(t)
       }, 1)
     })
-    store.uprogress.status = "processing"
+    uprogress.status = "processing"
     ws.send(data)
   }
 })
@@ -219,7 +210,7 @@ const worker = async () => {
     if (f.cloudPos === f.file.size) upqueue.shift()
   }
   if (upqueue.length) startWorker()
-  store.uprogress.status = "idle"
+  uprogress.status = "idle"
   workerRunning = false
 }
 let workerRunning: any = false
@@ -242,3 +233,12 @@ onUnmounted(() => {
   removeEventListener('drop', uploadHandler)
 })
 </script>
+<template>
+  <template>
+    <input ref="fileInput" @change="uploadHandler" type="file" multiple>
+    <input ref="folderInput" @change="uploadHandler" type="file" webkitdirectory>
+  </template>
+  <SvgButton name="add-file" data-tooltip="Upload files" @click="fileInput.click()" />
+  <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderInput.click()" />
+  <TransferBar :status=uprogress @cancel=cancelUploads />
+</template>

frontend/src/stores/main.ts

@@ -19,8 +19,6 @@ export const useMainStore = defineStore({
     cursor: '' as string,
     server: {} as Record<string, any>,
     dialog: '' as '' | 'login' | 'settings',
-    uprogress: {} as any,
-    dprogress: {} as any,
     prefs: {
       gallery: false,
       sortListing: '' as SortOrder,
@@ -91,13 +89,7 @@ export const useMainStore = defineStore({
     },
     focusBreadcrumb() {
       (document.querySelector('.breadcrumb') as HTMLAnchorElement).focus()
-    },
-    cancelDownloads() {
-      location.reload()  // FIXME
-    },
-    cancelUploads() {
-      location.reload()  // FIXME
-    },
+    }
   },
   getters: {
     sortOrder(): SortOrder { return this.query ? this.prefs.sortFiltered : this.prefs.sortListing },

frontend/vite.config.ts

@@ -4,6 +4,7 @@ import { defineConfig } from 'vite'
 import vue from '@vitejs/plugin-vue'
 
 // @ts-ignore
+import pluginRewriteAll from 'vite-plugin-rewrite-all'
 import svgLoader from 'vite-svg-loader'
 import Components from 'unplugin-vue-components/vite'
 
@@ -20,6 +21,7 @@ const dev_backend = {
 export default defineConfig({
   plugins: [
     vue(),
+    pluginRewriteAll(),
     svgLoader(),          // import svg files
     Components(),         // auto import components
   ],

pyproject.toml

@@ -27,7 +27,6 @@ dependencies = [
     "pyjwt",
     "pymupdf",
     "sanic",
-    "setproctitle",
     "stream-zip",
     "tomli_w",
 ]

scripts/build-frontend.py

@@ -1,8 +1,5 @@
 # noqa: INP001
-import os
-import shutil
 import subprocess
-from sys import stderr
 
 from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 
@@ -10,21 +7,6 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 class CustomBuildHook(BuildHookInterface):
     def initialize(self, version, build_data):
         super().initialize(version, build_data)
-        # A hack to stop building twice on run
-        if not build_data.get("force_include"):
-            return
-        stderr.write(">>> Building Cista frontend\n")
-        npm = shutil.which("npm")
-        if npm is None:
-            raise RuntimeError(
-                "NodeJS `npm` is required for building Cista but it was not found"
-            )
-        # npm --prefix doesn't work on Windows, so we chdir instead
-        os.chdir("frontend")
-        try:
-            stderr.write("### npm install\n")
-            subprocess.run([npm, "install"], check=True)  # noqa: S603
-            stderr.write("\n### npm run build\n")
-            subprocess.run([npm, "run", "build"], check=True)  # noqa: S603
-        finally:
-            os.chdir("..")
+        print("Building Cista frontend...")
+        subprocess.run("npm install --prefix frontend".split(" "), check=True)  # noqa: S603
+        subprocess.run("npm run build --prefix frontend".split(" "), check=True)  # noqa: S603