Pasteing files and folders to upload.

This is a major upgrade with assorted things included.

- Navigation flows improved, search appears in URL history, cleared when navigating to another folder
- More efficient file list format for faster loads
- Efficient updates, never re-send full root another time (except at connection)
- Large number of watching and filelist updates (inotify issues remain)
- File size coloring
- Fixed ZIP generation random glitches (thread race condition)
- Code refactoring, cleanup, typing fixes
- More tests

Reviewed-on: #3

cista/api.py

@@ -37,16 +37,23 @@ async def upload(req, ws):
             )
         req = msgspec.json.decode(text, type=FileRange)
         pos = req.start
-        data = None
-        while pos < req.end and (data := await ws.recv()) and isinstance(data, bytes):
+        while True:
+            data = await ws.recv()
+            if not isinstance(data, bytes):
+                break
+            if len(data) > req.end - pos:
+                raise ValueError(
+                    f"Expected up to {req.end - pos} bytes, got {len(data)} bytes"
+                )
             sentsize = await alink(("upload", req.name, pos, data, req.size))
             pos += typing.cast(int, sentsize)
+            if pos >= req.end:
+                break
         if pos != req.end:
             d = f"{len(data)} bytes" if isinstance(data, bytes) else data
             raise ValueError(f"Expected {req.end - pos} more bytes, got {d}")
         # Report success
         res = StatusMsg(status="ack", req=req)
-        print("ack", res)
         await asend(ws, res)
 
 

cista/fileio.py

@@ -34,9 +34,11 @@ class File:
             self.open_rw()
         assert self.fd is not None
         if file_size is not None:
+            assert pos + len(buffer) <= file_size
             os.ftruncate(self.fd, file_size)
-        os.lseek(self.fd, pos, os.SEEK_SET)
-        os.write(self.fd, buffer)
+        if buffer:
+            os.lseek(self.fd, pos, os.SEEK_SET)
+            os.write(self.fd, buffer)
 
     def __getitem__(self, slice):
         if self.fd is None:

cista/serve.py

@@ -1,6 +1,6 @@
 import os
 import re
-from pathlib import Path, PurePath
+from pathlib import Path
 
 from sanic import Sanic
 
@@ -15,7 +15,6 @@ def run(*, dev=False):
     # Silence Sanic's warning about running in production rather than debug
     os.environ["SANIC_IGNORE_PRODUCTION_WARNING"] = "1"
     confdir = config.conffile.parent
-    wwwroot = PurePath(__file__).parent / "wwwroot"
     if opts.get("ssl"):
         # Run plain HTTP redirect/acme server on port 80
         server80.app.prepare(port=80, motd=False)
@@ -27,7 +26,7 @@ def run(*, dev=False):
         motd=False,
         dev=dev,
         auto_reload=dev,
-        reload_dir={confdir, wwwroot},
+        reload_dir={confdir},
         access_log=True,
     )  # type: ignore
     if dev:

frontend/src/App.vue

@@ -44,8 +44,6 @@ watchEffect(() => {
 onMounted(loadSession)
 onMounted(watchConnect)
 onUnmounted(watchDisconnect)
-// Update human-readable x seconds ago messages from mtimes
-setInterval(documentStore.updateModified, 1000)
 const headerMain = ref<typeof HeaderMain | null>(null)
 let vert = 0
 let timer: any = null

frontend/src/components/FileExplorer.vue

@@ -79,7 +79,7 @@
 </template>
 
 <script setup lang="ts">
-import { ref, computed, watchEffect } from 'vue'
+import { ref, computed, watchEffect, onMounted, onUnmounted } from 'vue'
 import { useDocumentStore } from '@/stores/documents'
 import type { Document } from '@/repositories/Document'
 import FileRenameInput from './FileRenameInput.vue'
@@ -229,6 +229,13 @@ watchEffect(() => {
     focusBreadcrumb()
   }
 })
+// Update human-readable x seconds ago messages from mtimes
+let modifiedTimer: any = null
+const updateModified = () => {
+  for (const doc of props.documents) doc.modified = formatUnixDate(doc.mtime)
+}
+onMounted(() => { updateModified(); modifiedTimer = setInterval(updateModified, 1000) })
+onUnmounted(() => { clearInterval(modifiedTimer) })
 const mkdir = (doc: Document, name: string) => {
   const control = connect(controlUrl, {
     open() {

frontend/src/components/FileViewer.vue

@@ -1,52 +0,0 @@
-<template>
-  <object
-    v-if="props.type === 'pdf'"
-    :data="dataURL"
-    type="application/pdf"
-    width="100%"
-    height="100%"
-  ></object>
-  <a-image
-    v-else-if="props.type === 'image'"
-    width="50%"
-    :src="dataURL"
-    @click="() => setVisible(true)"
-    :previewMask="false"
-    :preview="{
-      visibleImg,
-      onVisibleChange: setVisible
-    }"
-  />
-  <!-- Unknown case -->
-  <h1 v-else>Unsupported file type</h1>
-</template>
-
-<script setup lang="ts">
-import { watchEffect, ref } from 'vue'
-import Router from '@/router/index'
-import { url_document_get } from '@/repositories/Document'
-
-const dataURL = ref('')
-watchEffect(() => {
-  dataURL.value = new URL(
-    url_document_get + Router.currentRoute.value.path,
-    location.origin
-  ).toString()
-})
-const emit = defineEmits({
-  visibleImg(value: boolean) {
-    return value
-  }
-})
-
-function setVisible(value: boolean) {
-  emit('visibleImg', value)
-}
-
-const props = defineProps<{
-  type?: string
-  visibleImg: boolean
-}>()
-</script>
-
-<style></style>

frontend/src/components/HeaderMain.vue

@@ -9,7 +9,7 @@
       <SvgButton
         name="create-folder"
         data-tooltip="New folder"
-        @click="() => documentStore.fileExplorer.newFolder()"
+        @click="() => documentStore.fileExplorer!.newFolder()"
       />
       <slot></slot>
       <div class="spacer smallgap"></div>
@@ -42,15 +42,15 @@ const showSearchInput = ref<boolean>(false)
 const search = ref<HTMLInputElement | null>()
 const searchButton = ref<HTMLButtonElement | null>()
 
-const closeSearch = ev => {
+const closeSearch = (ev: Event) => {
   if (!showSearchInput.value) return  // Already closing
   showSearchInput.value = false
   const breadcrumb = document.querySelector('.breadcrumb') as HTMLElement
   breadcrumb.focus()
   updateSearch(ev)
 }
-const updateSearch = ev => {
-  const q = ev.target.value
+const updateSearch = (ev: Event) => {
+  const q = (ev.target as HTMLInputElement).value
   let p = props.path.join('/')
   p = p ? `/${p}` : ''
   const url = q ? `${p}//${q}` : (p || '/')
@@ -58,9 +58,9 @@ const updateSearch = ev => {
   if (!props.query && q) router.push(url)
   else router.replace(url)
 }
-const toggleSearchInput = () => {
+const toggleSearchInput = (ev: Event) => {
   showSearchInput.value = !showSearchInput.value
-  if (!showSearchInput.value) return closeSearch()
+  if (!showSearchInput.value) return closeSearch(ev)
   nextTick(() => {
     const input = search.value
     if (input) input.focus()

frontend/src/components/HeaderSelected.vue

@@ -34,7 +34,7 @@ const op = (op: string, dst?: string) => {
   // @ts-ignore
   if (dst !== undefined) msg.dst = dst
   const control = connect(controlUrl, {
-    message(ev: WebSocmetMessageEvent) {
+    message(ev: MessageEvent) {
       const res = JSON.parse(ev.data)
       if ('error' in res) {
         console.error('Control socket error', msg, res.error)

frontend/src/components/NotificationLoading.vue

@@ -1,27 +0,0 @@
-<template>
-  <template v-for="upload in documentStore.uploadingDocuments" :key="upload.key">
-    <span>{{ upload.name }}</span>
-    <div class="progress-container">
-      <a-progress :percent="upload.progress" />
-      <CloseCircleOutlined class="close-button" @click="dismissUpload(upload.key)" />
-    </div>
-  </template>
-</template>
-<script setup lang="ts">
-import { useDocumentStore } from '@/stores/documents'
-const documentStore = useDocumentStore()
-
-function dismissUpload(key: number) {
-  documentStore.deleteUploadingDocument(key)
-}
-</script>
-
-<style scoped>
-.progress-container {
-  display: flex;
-  align-items: center;
-}
-.close-button:hover {
-  color: #b81414;
-}
-</style>

frontend/src/components/UploadButton.vue

@@ -11,29 +11,79 @@ const props = defineProps({
   path: Array<string>
 })
 
-
+type CloudFile = {
+  file: File
+  cloudName: string
+  cloudPos: number
+}
+function pasteHandler(event: ClipboardEvent) {
+  const items = Array.from(event.clipboardData?.items ?? [])
+  const infiles = [] as File[]
+  const dirs = [] as FileSystemDirectoryEntry[]
+  for (const item of items) {
+    if (item.kind !== 'file') continue
+    const entry = item.webkitGetAsEntry()
+    if (entry?.isFile) {
+      const file = item.getAsFile()
+      infiles.push(file)
+    } else if (entry?.isDirectory) {
+      dirs.push(entry as FileSystemDirectoryEntry)
+    }
+  }
+  if (infiles.length || dirs.length) {
+    event.preventDefault()
+    uploadFiles(infiles)
+    for (const entry of dirs) pasteDirectory(entry, `${props.path!.join('/')}/${entry.name}`)
+  }
+}
+const pasteDirectory = async (entry: FileSystemDirectoryEntry, loc: string) => {
+  const reader = entry.createReader()
+  const entries = await new Promise<any[]>(resolve => reader.readEntries(resolve))
+  const cloudfiles = [] as CloudFile[]
+  for (const entry of entries) {
+    const cloudName = `${loc}/${entry.name}`
+    if (entry.isFile) {
+      const file = await new Promise(resolve => entry.file(resolve)) as File
+      cloudfiles.push({file, cloudName, cloudPos: 0})
+    } else if (entry.isDirectory) {
+      await pasteDirectory(entry, cloudName)
+    }
+  }
+  if (cloudfiles.length) uploadCloudFiles(cloudfiles)
+}
 function uploadHandler(event: Event) {
   event.preventDefault()
-  event.stopPropagation()
   // @ts-ignore
-  let infiles = Array.from(event.dataTransfer?.files || event.target.files) as File[]
-  if (!infiles.length) return
+  const input = event.target as HTMLInputElement | null
+  const infiles = Array.from((input ?? (event as DragEvent).dataTransfer)?.files ?? []) as File[]
+  if (input) input.value = ''
+  if (infiles.length) uploadFiles(infiles)
+}
+
+const uploadFiles = (infiles: File[]) => {
   const loc = props.path!.join('/')
-  for (const f of infiles) {
-    f.cloudName = loc + '/' + (f.webkitRelativePath || f.name)
-    f.cloudPos = 0
+  let files = []
+  for (const file of infiles) {
+    files.push({
+      file,
+      cloudName: loc + '/' + (file.webkitRelativePath || file.name),
+      cloudPos: 0,
+    })
   }
-  const dotfiles = infiles.filter(f => f.cloudName.includes('/.'))
+  uploadCloudFiles(files)
+}
+const uploadCloudFiles = (files: CloudFile[]) => {
+  const dotfiles = files.filter(f => f.cloudName.includes('/.'))
   if (dotfiles.length) {
     documentStore.error = "Won't upload dotfiles"
     console.log("Dotfiles omitted", dotfiles)
-    infiles = infiles.filter(f => !f.cloudName.includes('/.'))
+    files = files.filter(f => !f.cloudName.includes('/.'))
   }
-  if (!infiles.length) return
-  infiles.sort((a, b) => collator.compare(a.cloudName, b.cloudName))
+  if (!files.length) return
+  files.sort((a, b) => collator.compare(a.cloudName, b.cloudName))
   // @ts-ignore
-  upqueue = upqueue.concat(infiles)
-  statsAdd(infiles)
+  upqueue = [...upqueue, ...files]
+  statsAdd(files)
   startWorker()
 }
 
@@ -49,13 +99,14 @@ const uprogress_init = {
   tlast: 0,
   statbytes: 0,
   statdur: 0,
-  files: [],
+  files: [] as CloudFile[],
   filestart: 0,
   fileidx: 0,
   filecount: 0,
   filename: '',
   filesize: 0,
   filepos: 0,
+  status: 'idle',
 }
 const uprogress = reactive({...uprogress_init})
 const percent = computed(() => uprogress.uploaded / uprogress.total * 100)
@@ -66,7 +117,7 @@ const speed = computed(() => {
   if (tsince > 1 / s) return 1 / tsince  // Next block is late or not coming, decay
   return s  // "Current speed"
 })
-const speeddisp = computed(() => speed.value ? speed.value.toFixed(speed.value < 100 ? 1 : 0) + '\u202FMB/s': 'stalled')
+const speeddisp = computed(() => speed.value ? speed.value.toFixed(speed.value < 10 ? 1 : 0) + '\u202FMB/s': 'stalled')
 setInterval(() => {
   if (Date.now() - uprogress.tlast > 3000) {
     // Reset
@@ -78,7 +129,7 @@ setInterval(() => {
     uprogress.statdur *= .9
   }
 }, 100)
-const statUpdate = ({name, size, start, end}) => {
+const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => {
   if (name !== uprogress.filename) return  // If stats have been reset
   const now = Date.now()
   uprogress.uploaded = uprogress.filestart + end
@@ -97,7 +148,7 @@ const statNextFile = () => {
   const f = uprogress.files.shift()
   if (!f) return statReset()
   uprogress.filepos = 0
-  uprogress.filesize = f.size
+  uprogress.filesize = f.file.size
   uprogress.filename = f.cloudName
 }
 const statReset = () => {
@@ -105,14 +156,14 @@ const statReset = () => {
   uprogress.t0 = Date.now()
   uprogress.tlast = uprogress.t0 + 1
 }
-const statsAdd = (f: Array<File>) => {
+const statsAdd = (f: CloudFile[]) => {
   if (uprogress.files.length === 0) statReset()
-  uprogress.total += f.reduce((a, b) => a + b.size, 0)
+  uprogress.total += f.reduce((a, b) => a + b.file.size, 0)
   uprogress.filecount += f.length
-  uprogress.files = uprogress.files.concat(f)
+  uprogress.files = [...uprogress.files, ...f]
   statNextFile()
 }
-let upqueue = [] as File[]
+let upqueue = [] as CloudFile[]
 
 // TODO: Rewrite as WebSocket class
 const WSCreate = async () => await new Promise<WebSocket>(resolve => {
@@ -155,18 +206,17 @@ const worker = async () => {
   const ws = await WSCreate()
   while (upqueue.length) {
     const f = upqueue[0]
-    if (f.cloudPos === f.size) {
-      upqueue.shift()
-      continue
-    }
     const start = f.cloudPos
-    const end = Math.min(f.size, start + (1<<20))
-    const control = { name: f.cloudName, size: f.size, start, end }
-    const data = f.slice(start, end)
+    const end = Math.min(f.file.size, start + (1<<20))
+    const control = { name: f.cloudName, size: f.file.size, start, end }
+    const data = f.file.slice(start, end)
     f.cloudPos = end
     // Note: files may get modified during I/O
+    // @ts-ignore FIXME proper WebSocket class, avoid attaching functions to WebSocket object
     ws.sendMsg(control)
+    // @ts-ignore
     await ws.sendData(data)
+    if (f.cloudPos === f.file.size) upqueue.shift()
   }
   if (upqueue.length) startWorker()
   uprogress.status = "idle"
@@ -184,8 +234,10 @@ onMounted(() => {
   // Need to prevent both to prevent browser from opening the file
   addEventListener('dragover', uploadHandler)
   addEventListener('drop', uploadHandler)
+  addEventListener('paste', pasteHandler)
 })
 onUnmounted(() => {
+  removeEventListener('paste', pasteHandler)
   removeEventListener('dragover', uploadHandler)
   removeEventListener('drop', uploadHandler)
 })
@@ -207,7 +259,7 @@ onUnmounted(() => {
           {{ (uprogress.filepos / uprogress.filesize * 100).toFixed(0) + '\u202F%' }}
         </span>
       </span>
-      <span class="position" v-if="uprogress.filesize > 1e7">
+      <span class="position" v-if="uprogress.total > 1e7">
         {{ (uprogress.uploaded / 1e6).toFixed(0) + '\u202F/\u202F' + (uprogress.total / 1e6).toFixed(0) + '\u202FMB' }}
       </span>
       <span class="speed">{{ speeddisp }}</span>

frontend/src/repositories/WS.ts

@@ -109,7 +109,7 @@ const handleWatchMessage = (event: MessageEvent) => {
   }
 }
 
-function handleRootMessage({ root }: { root: DirEntry }) {
+function handleRootMessage({ root }: { root: FileEntry[] }) {
   const store = useDocumentStore()
   console.log('Watch root', root)
   store.updateRoot(root)

frontend/src/stores/documents.ts

@@ -21,9 +21,7 @@ export const useDocumentStore = defineStore({
   state: () => ({
     document: [] as Document[],
     selected: new Set<FUID>(),
-    uploadingDocuments: [],
-    uploadCount: 0 as number,
-    fileExplorer: null,
+    fileExplorer: null as any,
     error: '' as string,
     connected: false,
     server: {} as Record<string, any>,
@@ -53,12 +51,8 @@ export const useDocumentStore = defineStore({
         })
         loc.push(name)
       }
-      console.log("Documents", docs)
       this.document = docs as Document[]
     },
-    updateModified() {
-      for (const doc of this.document) doc.modified = formatUnixDate(doc.mtime)
-    },
     login(username: string, privileged: boolean) {
       this.user.username = username
       this.user.privileged = privileged