Better transparent Cista image

- Major memory usage reduction in video previews
- Finally builds properly on Windows too

Reviewed-on: #6

cista/__main__.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from pathlib import Path
 
@@ -61,6 +62,7 @@ def _main():
         path = None
     _confdir(args)
     exists = config.conffile.exists()
+    print(config.conffile, exists)
     import_droppy = args["--import-droppy"]
     necessary_opts = exists or import_droppy or path
     if not necessary_opts:
@@ -117,7 +119,8 @@ def _confdir(args):
                 raise ValueError("Config path is not a directory")
             # Accidentally pointed to the db.toml, use parent
             confdir = confdir.parent
-        config.conffile = confdir / config.conffile.name
+        os.environ["CISTA_HOME"] = confdir.as_posix()
+    config.init_confdir()  # Uses environ if available
 
 
 def _user(args):

cista/app.py

@@ -3,6 +3,7 @@ import datetime
 import mimetypes
 import threading
 from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import cpu_count
 from pathlib import Path, PurePath, PurePosixPath
 from stat import S_IFDIR, S_IFREG
 from urllib.parse import unquote
@@ -14,6 +15,7 @@ from blake3 import blake3
 from sanic import Blueprint, Sanic, empty, raw, redirect
 from sanic.exceptions import Forbidden, NotFound
 from sanic.log import logger
+from setproctitle import setproctitle
 from stream_zip import ZIP_AUTO, stream_zip
 
 from cista import auth, config, preview, session, watching
@@ -30,11 +32,16 @@ app.blueprint(bp)
 app.exception(Exception)(handle_sanic_exception)
 
 
+setproctitle("cista-main")
+
+
 @app.before_server_start
 async def main_start(app, loop):
     config.load_config()
+    setproctitle(f"cista {config.config.path.name}")
+    workers = max(2, min(8, cpu_count()))
     app.ctx.threadexec = ThreadPoolExecutor(
-        max_workers=8, thread_name_prefix="cista-ioworker"
+        max_workers=workers, thread_name_prefix="cista-ioworker"
     )
     await watching.start(app, loop)
 

cista/config.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import os
 import secrets
 import sys
+from contextlib import suppress
 from functools import wraps
 from hashlib import sha256
 from pathlib import Path, PurePath
@@ -33,7 +35,23 @@ class Link(msgspec.Struct, omit_defaults=True):
 
 
 config = None
-conffile = Path.home() / ".local/share/cista/db.toml"
+conffile = None
+
+
+def init_confdir():
+    if p := os.environ.get("CISTA_HOME"):
+        home = Path(p)
+    else:
+        xdg = os.environ.get("XDG_CONFIG_HOME")
+        home = (
+            Path(xdg).expanduser() / "cista" if xdg else Path.home() / ".config/cista"
+        )
+    if not home.is_dir():
+        home.mkdir(parents=True, exist_ok=True)
+        home.chmod(0o700)
+
+    global conffile
+    conffile = home / "db.toml"
 
 
 def derived_secret(*params, len=8) -> bytes:
@@ -61,8 +79,8 @@ def dec_hook(typ, obj):
 
 def config_update(modify):
     global config
-    if not conffile.exists():
+    if conffile is None:
-        conffile.parent.mkdir(parents=True, exist_ok=True)
+        init_confdir()
     tmpname = conffile.with_suffix(".tmp")
     try:
         f = tmpname.open("xb")
@@ -76,10 +94,6 @@ def config_update(modify):
             old = conffile.read_bytes()
             c = msgspec.toml.decode(old, type=Config, dec_hook=dec_hook)
         except FileNotFoundError:
-            # No existing config file, make sure we have a folder...
-            confdir = conffile.parent
-            confdir.mkdir(parents=True, exist_ok=True)
-            confdir.chmod(0o700)
             old = b""
             c = None
         c = modify(c)
@@ -92,7 +106,9 @@ def config_update(modify):
         f.write(new)
         f.close()
         if sys.platform == "win32":
-            conffile.unlink()  # Windows doesn't support atomic replace
+            # Windows doesn't support atomic replace
+            with suppress(FileNotFoundError):
+                conffile.unlink()
         tmpname.rename(conffile)  # Atomic replace
     except:
         f.close()
@@ -120,6 +136,8 @@ def modifies_config(modify):
 
 def load_config():
     global config
+    if conffile is None:
+        init_confdir()
     config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook)
 
 

cista/preview.py

@@ -1,4 +1,5 @@
 import asyncio
+import gc
 import io
 import mimetypes
 import urllib.parse
@@ -17,6 +18,8 @@ from sanic.log import logger
 from cista import config
 from cista.util.filename import sanitize
 
+DISPLAYMATRIX = av.stream.SideData.DISPLAYMATRIX
+
 bp = Blueprint("preview", url_prefix="/preview")
 
 
@@ -96,19 +99,19 @@ def process_pdf(path, *, maxsize, maxzoom, quality, page_number=0):
 def process_video(path, *, maxsize, quality):
     with av.open(str(path)) as container:
         stream = container.streams.video[0]
-        rotation = (
-            stream.side_data
-            and stream.side_data.get(av.stream.SideData.DISPLAYMATRIX)
-            or 0
-        )
         stream.codec_context.skip_frame = "NONKEY"
+        rot = stream.side_data and stream.side_data.get(DISPLAYMATRIX) or 0
         container.seek(container.duration // 8)
-        frame = next(container.decode(stream))
+        img = next(container.decode(stream)).to_image()
-        img = frame.to_image()
+        del stream
 
     img.thumbnail((maxsize, maxsize))
     imgdata = io.BytesIO()
-    if rotation:
+    if rot:
-        img = img.rotate(rotation, expand=True)
+        img = img.rotate(rot, expand=True)
     img.save(imgdata, format="webp", quality=quality, method=4)
-    return imgdata.getvalue()
+    del img
+    ret = imgdata.getvalue()
+    del imgdata
+    gc.collect()
+    return ret

cista/serve.py

@@ -26,7 +26,6 @@ def run(*, dev=False):
         motd=False,
         dev=dev,
         auto_reload=dev,
-        reload_dir={confdir},
         access_log=True,
     )  # type: ignore
     if dev:

frontend/.npmrc

@@ -0,0 +1,2 @@
+audit=false
+fund=false

frontend/package.json

@@ -12,6 +12,9 @@
     "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore",
     "format": "prettier --write src/"
   },
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "dependencies": {
     "@imengyu/vue3-context-menu": "^1.3.3",
     "@vueuse/core": "^10.4.1",
@@ -21,7 +24,6 @@
     "pinia": "^2.1.6",
     "pinia-plugin-persistedstate": "^3.2.0",
     "unplugin-vue-components": "^0.25.2",
-    "vite-plugin-rewrite-all": "^1.0.1",
     "vite-svg-loader": "^4.0.0",
     "vue": "^3.3.4",
     "vue-router": "^4.2.4"

frontend/src/App.vue

@@ -10,6 +10,10 @@
   <main>
     <RouterView :path="path.pathList" :query="path.query" />
   </main>
+  <footer>
+    <TransferBar :status=store.uprogress @cancel=store.cancelUploads class=upload />
+    <TransferBar :status=store.dprogress @cancel=store.cancelDownloads class=download />
+  </footer>
 </template>
 
 <script setup lang="ts">

frontend/src/components/DownloadButton.vue

@@ -1,6 +1,5 @@
 <template>
   <SvgButton name="download" data-tooltip="Download" @click="download" />
-  <TransferBar :status=progress @cancel=cancelDownloads />
 </template>
 
 <script setup lang="ts">
@@ -26,22 +25,22 @@ const status_init = {
   filepos: 0,
   status: 'idle',
 }
-const progress = reactive({...status_init})
+store.dprogress = {...status_init}
 setInterval(() => {
-  if (Date.now() - progress.tlast > 3000) {
+  if (Date.now() - store.dprogress.tlast > 3000) {
     // Reset
-    progress.statbytes = 0
+    store.dprogress.statbytes = 0
-    progress.statdur = 1
+    store.dprogress.statdur = 1
   } else {
     // Running average by decay
-    progress.statbytes *= .9
+    store.dprogress.statbytes *= .9
-    progress.statdur *= .9
+    store.dprogress.statdur *= .9
   }
 }, 100)
 const statReset = () => {
-  Object.assign(progress, status_init)
+  Object.assign(store.dprogress, status_init)
-  progress.t0 = Date.now()
+  store.dprogress.t0 = Date.now()
-  progress.tlast = progress.t0 + 1
+  store.dprogress.tlast = store.dprogress.t0 + 1
 }
 const cancelDownloads = () => {
   location.reload()  // FIXME
@@ -61,9 +60,9 @@ const filesystemdl = async (sel: SelectedItems, handle: FileSystemDirectoryHandl
   console.log('Downloading to filesystem', sel.recursive)
   for (const [rel, full, doc] of sel.recursive) {
     if (doc.dir) continue
-    progress.files.push(rel)
+    store.dprogress.files.push(rel)
-    ++progress.filecount
+    ++store.dprogress.filecount
-    progress.total += doc.size
+    store.dprogress.total += doc.size
   }
   for (const [rel, full, doc] of sel.recursive) {
     // Create any missing directories
@@ -73,6 +72,7 @@ const filesystemdl = async (sel: SelectedItems, handle: FileSystemDirectoryHandl
     }
     const r = rel.slice(hdir.length)
     for (const dir of r.split('/').slice(0, doc.dir ? undefined : -1)) {
+      if (!dir) continue
       hdir += `${dir}/`
       try {
         h = await h.getDirectoryHandle(dir.normalize('NFC'), { create: true })
@@ -101,22 +101,22 @@ const filesystemdl = async (sel: SelectedItems, handle: FileSystemDirectoryHandl
       throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`)
     }
     if (res.body) {
-      ++progress.fileidx
+      ++store.dprogress.fileidx
       const reader = res.body.getReader()
       await writable.truncate(0)
       store.error = "Direct download."
-      progress.tlast = Date.now()
+      store.dprogress.tlast = Date.now()
       while (true) {
         const { value, done } = await reader.read()
         if (done) break
         await writable.write(value)
         const now = Date.now()
         const size = value.byteLength
-        progress.xfer += size
+        store.dprogress.xfer += size
-        progress.filepos += size
+        store.dprogress.filepos += size
-        progress.statbytes += size
+        store.dprogress.statbytes += size
-        progress.statdur += now - progress.tlast
+        store.dprogress.statdur += now - store.dprogress.tlast
-        progress.tlast = now
+        store.dprogress.tlast = now
       }
     }
     await writable.close()

frontend/src/components/FileExplorer.vue

@@ -115,6 +115,7 @@ const rename = (doc: Doc, newName: string) => {
 }
 defineExpose({
   newFolder() {
+    console.log("New folder")
     const now = Math.floor(Date.now() / 1000)
     editing.value = new Doc({
       loc: loc.value,
@@ -124,6 +125,7 @@ defineExpose({
       mtime: now,
       size: 0,
     })
+    store.cursor = editing.value.key
   },
   toggleSelectAll() {
     console.log('Select')

frontend/src/components/Gallery.vue

@@ -2,8 +2,11 @@
   <div v-if="props.documents.length || editing" class="gallery" ref="gallery">
     <GalleryFigure v-if="editing?.key === 'new'" :doc="editing" :key=editing.key :editing="{rename: mkdir, exit}" />
     <template v-for="(doc, index) in documents" :key=doc.key>
-      <GalleryFigure :doc=doc :editing="editing === doc ? {rename, exit} : null">
+      <GalleryFigure :doc=doc :editing="editing === doc ? {rename, exit} : null" @menu="contextMenu($event, doc)">
-        <BreadCrumb v-if=showFolderBreadcrumb(index) :path="doc.loc ? doc.loc.split('/') : []" class="folder-change"/>
+        <template v-if=showFolderBreadcrumb(index)>
+          <BreadCrumb :path="doc.loc ? doc.loc.split('/') : []" class="folder-change"/>
+          <div class="spacer"></div>
+        </template>
       </GalleryFigure>
     </template>
   </div>
@@ -67,6 +70,7 @@ defineExpose({
       mtime: now,
       size: 0,
     })
+    store.cursor = editing.value.key
   },
   toggleSelectAll() {
     console.log('Select')
@@ -254,6 +258,9 @@ const contextMenu = (ev: MouseEvent, doc: Doc) => {
   align-items: end;
 }
 .breadcrumb {
-  border-radius: .5em;
+  border-radius: .5em 0 0 .5em;
+}
+.spacer {
+  flex: 0 1000000000 4rem;
 }
 </style>

frontend/src/components/GalleryFigure.vue

@@ -9,7 +9,7 @@
       <slot></slot>
       <MediaPreview ref=m :doc="doc" tabindex=-1 quality="sz=512" class="figcontent" />
       <div class="titlespacer"></div>
-      <figcaption @click.prevent>
+      <figcaption @click.prevent @contextmenu.prevent="$emit('menu', $event)">
         <template v-if="editing">
           <FileRenameInput :doc=doc :rename=editing.rename :exit=editing.exit />
         </template>

frontend/src/components/HeaderMain.vue

@@ -8,7 +8,7 @@
     <SvgButton
       name="create-folder"
       data-tooltip="New folder"
-      @click="() => store.fileExplorer!.newFolder()"
+      @click="() => {  console.log('New', store.fileExplorer); store.fileExplorer!.newFolder(); console.log('Done')}"
     />
     <slot></slot>
     <div class="spacer smallgap"></div>

frontend/src/components/TransferBar.vue

@@ -57,13 +57,12 @@ const speeddisp = computed(() => speed.value ? speed.value.toFixed(speed.value <
   display: flex;
   flex-direction: column;
   color: var(--primary-color);
-  position: fixed;
+  width: 100%;
-  left: 0;
-  bottom: 0;
-  width: 100vw;
 }
 .statustext {
   display: flex;
+  align-items: center;
+  margin: 0 .5em;
   padding: 0.5rem 0;
 }
 span {
@@ -84,4 +83,12 @@ span {
 .position { min-width: 4em }
 .speed { min-width: 4em }
 
+.upload .statustext::before {
+  font-size: 1.5em;
+  content: '🔺'
+}
+.download .statustext::before {
+  font-size: 1.5em;
+  content: '🔻'
+}
 </style>

frontend/src/components/UploadButton.vue

@@ -1,3 +1,12 @@
+<template>
+  <template>
+    <input ref="fileInput" @change="uploadHandler" type="file" multiple>
+    <input ref="folderInput" @change="uploadHandler" type="file" webkitdirectory>
+  </template>
+  <SvgButton name="add-file" data-tooltip="Upload files" @click="fileInput.click()" />
+  <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderInput.click()" />
+</template>
+
 <script setup lang="ts">
 import { connect, uploadUrl } from '@/repositories/WS';
 import { useMainStore } from '@/stores/main'
@@ -108,50 +117,50 @@ const uprogress_init = {
   filepos: 0,
   status: 'idle',
 }
-const uprogress = reactive({...uprogress_init})
+store.uprogress = {...uprogress_init}
 setInterval(() => {
-  if (Date.now() - uprogress.tlast > 3000) {
+  if (Date.now() - store.uprogress.tlast > 3000) {
     // Reset
-    uprogress.statbytes = 0
+    store.uprogress.statbytes = 0
-    uprogress.statdur = 1
+    store.uprogress.statdur = 1
   } else {
     // Running average by decay
-    uprogress.statbytes *= .9
+    store.uprogress.statbytes *= .9
-    uprogress.statdur *= .9
+    store.uprogress.statdur *= .9
   }
 }, 100)
 const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => {
-  if (name !== uprogress.filename) return  // If stats have been reset
+  if (name !== store.uprogress.filename) return  // If stats have been reset
   const now = Date.now()
-  uprogress.xfer = uprogress.filestart + end
+  store.uprogress.xfer = store.uprogress.filestart + end
-  uprogress.filepos = end
+  store.uprogress.filepos = end
-  uprogress.statbytes += end - start
+  store.uprogress.statbytes += end - start
-  uprogress.statdur += now - uprogress.tlast
+  store.uprogress.statdur += now - store.uprogress.tlast
-  uprogress.tlast = now
+  store.uprogress.tlast = now
   // File finished?
   if (end === size) {
-    uprogress.filestart += size
+    store.uprogress.filestart += size
     statNextFile()
-    if (++uprogress.fileidx >= uprogress.filecount) statReset()
+    if (++store.uprogress.fileidx >= store.uprogress.filecount) statReset()
   }
 }
 const statNextFile = () => {
-  const f = uprogress.files.shift()
+  const f = store.uprogress.files.shift()
   if (!f) return statReset()
-  uprogress.filepos = 0
+  store.uprogress.filepos = 0
-  uprogress.filesize = f.file.size
+  store.uprogress.filesize = f.file.size
-  uprogress.filename = f.cloudName
+  store.uprogress.filename = f.cloudName
 }
 const statReset = () => {
-  Object.assign(uprogress, uprogress_init)
+  Object.assign(store.uprogress, uprogress_init)
-  uprogress.t0 = Date.now()
+  store.uprogress.t0 = Date.now()
-  uprogress.tlast = uprogress.t0 + 1
+  store.uprogress.tlast = store.uprogress.t0 + 1
 }
 const statsAdd = (f: CloudFile[]) => {
-  if (uprogress.files.length === 0) statReset()
+  if (store.uprogress.files.length === 0) statReset()
-  uprogress.total += f.reduce((a, b) => a + b.file.size, 0)
+  store.uprogress.total += f.reduce((a, b) => a + b.file.size, 0)
-  uprogress.filecount += f.length
+  store.uprogress.filecount += f.length
-  uprogress.files = [...uprogress.files, ...f]
+  store.uprogress.files = [...store.uprogress.files, ...f]
   statNextFile()
 }
 let upqueue = [] as CloudFile[]
@@ -181,7 +190,7 @@ const WSCreate = async () => await new Promise<WebSocket>(resolve => {
   // @ts-ignore
   ws.sendData = async (data: any) => {
     // Wait until the WS is ready to send another message
-    uprogress.status = "uploading"
+    store.uprogress.status = "uploading"
     await new Promise(resolve => {
       const t = setInterval(() => {
         if (ws.bufferedAmount > 1<<20) return
@@ -189,7 +198,7 @@ const WSCreate = async () => await new Promise<WebSocket>(resolve => {
         clearInterval(t)
       }, 1)
     })
-    uprogress.status = "processing"
+    store.uprogress.status = "processing"
     ws.send(data)
   }
 })
@@ -210,7 +219,7 @@ const worker = async () => {
     if (f.cloudPos === f.file.size) upqueue.shift()
   }
   if (upqueue.length) startWorker()
-  uprogress.status = "idle"
+  store.uprogress.status = "idle"
   workerRunning = false
 }
 let workerRunning: any = false
@@ -233,12 +242,3 @@ onUnmounted(() => {
   removeEventListener('drop', uploadHandler)
 })
 </script>
-<template>
-  <template>
-    <input ref="fileInput" @change="uploadHandler" type="file" multiple>
-    <input ref="folderInput" @change="uploadHandler" type="file" webkitdirectory>
-  </template>
-  <SvgButton name="add-file" data-tooltip="Upload files" @click="fileInput.click()" />
-  <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderInput.click()" />
-  <TransferBar :status=uprogress @cancel=cancelUploads />
-</template>

frontend/src/stores/main.ts

@@ -19,6 +19,8 @@ export const useMainStore = defineStore({
     cursor: '' as string,
     server: {} as Record<string, any>,
     dialog: '' as '' | 'login' | 'settings',
+    uprogress: {} as any,
+    dprogress: {} as any,
     prefs: {
       gallery: false,
       sortListing: '' as SortOrder,
@@ -89,7 +91,13 @@ export const useMainStore = defineStore({
     },
     focusBreadcrumb() {
       (document.querySelector('.breadcrumb') as HTMLAnchorElement).focus()
-    }
+    },
+    cancelDownloads() {
+      location.reload()  // FIXME
+    },
+    cancelUploads() {
+      location.reload()  // FIXME
+    },
   },
   getters: {
     sortOrder(): SortOrder { return this.query ? this.prefs.sortFiltered : this.prefs.sortListing },

frontend/src/views/ExplorerView.vue

@@ -27,7 +27,6 @@ import Router from '@/router/index'
 import { needleFormat, localeIncludes, collator } from '@/utils'
 import { sorted } from '@/utils/docsort'
 import FileExplorer from '@/components/FileExplorer.vue'
-import cog from '@/assets/svg/cog.svg'
 
 const store = useMainStore()
 const fileExplorer = ref()

frontend/vite.config.ts

@@ -4,7 +4,6 @@ import { defineConfig } from 'vite'
 import vue from '@vitejs/plugin-vue'
 
 // @ts-ignore
-import pluginRewriteAll from 'vite-plugin-rewrite-all'
 import svgLoader from 'vite-svg-loader'
 import Components from 'unplugin-vue-components/vite'
 
@@ -21,7 +20,6 @@ const dev_backend = {
 export default defineConfig({
   plugins: [
     vue(),
-    pluginRewriteAll(),
     svgLoader(),          // import svg files
     Components(),         // auto import components
   ],

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "pyjwt",
     "pymupdf",
     "sanic",
+    "setproctitle",
     "stream-zip",
     "tomli_w",
 ]
@@ -48,7 +49,7 @@ source = "vcs"
 
 [tool.hatch.build]
 artifacts = ["cista/wwwroot"]
-hooks.custom.path = "scripts/build-frontend.py"
+targets.sdist.hooks.custom.path = "scripts/build-frontend.py"
 hooks.vcs.version-file = "cista/_version.py"
 hooks.vcs.template = """
 # This file is automatically generated by hatch build.

scripts/build-frontend.py

@@ -1,5 +1,8 @@
 # noqa: INP001
+import os
+import shutil
 import subprocess
+from sys import stderr
 
 from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 
@@ -7,6 +10,18 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 class CustomBuildHook(BuildHookInterface):
     def initialize(self, version, build_data):
         super().initialize(version, build_data)
-        print("Building Cista frontend...")
+        stderr.write(">>> Building Cista frontend\n")
-        subprocess.run("npm install --prefix frontend".split(" "), check=True)  # noqa: S603
+        npm = shutil.which("npm")
-        subprocess.run("npm run build --prefix frontend".split(" "), check=True)  # noqa: S603
+        if npm is None:
+            raise RuntimeError(
+                "NodeJS `npm` is required for building Cista but it was not found"
+            )
+        # npm --prefix doesn't work on Windows, so we chdir instead
+        os.chdir("frontend")
+        try:
+            stderr.write("### npm install\n")
+            subprocess.run([npm, "install"], check=True)  # noqa: S603
+            stderr.write("\n### npm run build\n")
+            subprocess.run([npm, "run", "build"], check=True)  # noqa: S603
+        finally:
+            os.chdir("..")