From a3a6b3771c32aa1d4c75913381aef570287fbf54 Mon Sep 17 00:00:00 2001
From: Leo Vasanko <leo@git>
Date: Wed, 8 Nov 2023 12:06:06 -0800
Subject: [PATCH] Rewritten uploads with queuing multiple tasks etc.

---
 cista-front/src/components/UploadButton.vue | 302 ++++++++++++--------
 1 file changed, 180 insertions(+), 122 deletions(-)

diff --git a/cista-front/src/components/UploadButton.vue b/cista-front/src/components/UploadButton.vue
index ccbcbaa..ef22c8e 100644
--- a/cista-front/src/components/UploadButton.vue
+++ b/cista-front/src/components/UploadButton.vue
@@ -1,155 +1,224 @@
 <script setup lang="ts">
 import { connect, uploadUrl } from '@/repositories/WS';
 import { useDocumentStore } from '@/stores/documents'
-import { computed, reactive, ref } from 'vue'
+import { collator } from '@/utils';
+import { computed, onMounted, onUnmounted, reactive, ref } from 'vue'
 
-const fileUploadButton = ref()
-const folderUploadButton = ref()
+const fileInput = ref()
+const folderInput = ref()
 const documentStore = useDocumentStore()
-const isNotificationOpen = ref(false)
 const props = defineProps({
   path: Array<string>
 })
 
+
+function uploadHandler(event: Event) {
+  event.preventDefault()
+  event.stopPropagation()
+  // @ts-ignore
+  let infiles = Array.from(event.dataTransfer?.files || event.target.files) as File[]
+  if (!infiles.length) return
+  const loc = props.path!.join('/')
+  for (const f of infiles) {
+    f.cloudName = loc + '/' + (f.webkitRelativePath || f.name)
+    f.cloudPos = 0
+  }
+  const dotfiles = infiles.filter(f => f.cloudName.includes('/.'))
+  if (dotfiles.length) {
+    documentStore.error = "Won't upload dotfiles"
+    console.log("Dotfiles omitted", dotfiles)
+    infiles = infiles.filter(f => !f.cloudName.includes('/.'))
+  }
+  if (!infiles.length) return
+  infiles.sort((a, b) => collator.compare(a.cloudName, b.cloudName))
+  // @ts-ignore
+  upqueue = upqueue.concat(infiles)
+  statsAdd(infiles)
+  startWorker()
+}
+
+const cancelUploads = () => {
+  upqueue = []
+  statReset()
+}
+
 const uprogress_init = {
   total: 0,
   uploaded: 0,
   t0: 0,
   tlast: 0,
-  filestart: [] as number[],
+  statbytes: 0,
+  statdur: 0,
+  files: [],
+  filestart: 0,
   fileidx: 0,
+  filecount: 0,
   filename: '',
-  filepos: 0,
   filesize: 0,
+  filepos: 0,
 }
 const uprogress = reactive({...uprogress_init})
-const speed = computed(() => uprogress.uploaded / (uprogress.tlast - uprogress.t0) / 1e3)
-const resetProgress = () => {
+const percent = computed(() => uprogress.uploaded / uprogress.total * 100)
+const speed = computed(() => {
+  let s = uprogress.statbytes / uprogress.statdur / 1e3
+  const tsince = (Date.now() - uprogress.tlast) / 1e3
+  if (tsince > 5 / s) return 0  // Less than fifth of previous speed => stalled
+  if (tsince > 1 / s) return 1 / tsince  // Next block is late or not coming, decay
+  return s  // "Current speed"
+})
+const speeddisp = computed(() => speed.value ? speed.value.toFixed(speed.value < 100 ? 1 : 0) + '\u202FMB/s': 'stalled')
+setInterval(() => {
+  if (Date.now() - uprogress.tlast > 3000) {
+    // Reset
+    uprogress.statbytes = 0
+    uprogress.statdur = 1
+  } else {
+    // Running average by decay
+    uprogress.statbytes *= .9
+    uprogress.statdur *= .9
+  }
+}, 100)
+const statUpdate = ({name, size, start, end}) => {
+  if (name !== uprogress.filename) return  // If stats have been reset
+  const now = Date.now()
+  uprogress.uploaded = uprogress.filestart + end
+  uprogress.filepos = end
+  uprogress.statbytes += end - start
+  uprogress.statdur += now - uprogress.tlast
+  uprogress.tlast = now
+  // File finished?
+  if (end === size) {
+    uprogress.filestart += size
+    statNextFile()
+    if (++uprogress.fileidx >= uprogress.filecount) statReset()
+  }
+}
+const statNextFile = () => {
+  const f = uprogress.files.shift()
+  if (!f) return statReset()
+  uprogress.filepos = 0
+  uprogress.filesize = f.size
+  uprogress.filename = f.cloudName
+}
+const statReset = () => {
   Object.assign(uprogress, uprogress_init)
   uprogress.t0 = Date.now()
   uprogress.tlast = uprogress.t0 + 1
 }
-async function uploadHandler(event: Event) {
-  const target = event.target as HTMLInputElement
-  if (!target?.files?.length) {
-    documentStore.error = 'No files selected'
-    return
-  }
-  const ws = await new Promise<WebSocket>(resolve => {
-    const ws = connect(uploadUrl, {
-      open(ev: Event) { resolve(ws) },
-      error(ev: Event) {
-        console.error('Upload socket error', ev)
-        documentStore.error = 'Upload socket error'
-      },
-      message(ev: MessageEvent) {
-        const res = JSON.parse(ev!.data)
-        if ('error' in res) {
-          console.error('Upload socket error', res.error)
-          documentStore.error = res.error.message
-          return
-        }
-        if (res.status === 'ack') {
-          // Upload progress upgrade on SERVER ACK
-          const fstart = uprogress.filestart[uprogress.fileidx]
-          uprogress.uploaded = fstart + res.req.end
-          uprogress.filepos = res.req.end
-          uprogress.tlast = Date.now()
-        } else console.log('Unknown upload response', res)
-      },
-    })
-    ws.binaryType = 'arraybuffer'
+const statsAdd = (f: Array<File>) => {
+  if (uprogress.files.length === 0) statReset()
+  uprogress.total += f.reduce((a, b) => a + b.size, 0)
+  uprogress.filecount += f.length
+  uprogress.files = uprogress.files.concat(f)
+  statNextFile()
+}
+let upqueue = [] as File[]
+
+// TODO: Rewrite as WebSocket class
+const WSCreate = async () => await new Promise<WebSocket>(resolve => {
+  const ws = connect(uploadUrl, {
+    open(ev: Event) { resolve(ws) },
+    error(ev: Event) {
+      console.error('Upload socket error', ev)
+      documentStore.error = 'Upload socket error'
+    },
+    message(ev: MessageEvent) {
+      const res = JSON.parse(ev!.data)
+      if ('error' in res) {
+        console.error('Upload socket error', res.error)
+        documentStore.error = res.error.message
+        return
+      }
+      if (res.status === 'ack') {
+        statUpdate(res.req)
+      } else console.log('Unknown upload response', res)
+    },
   })
-  async function wsFlush(limit: number) {
+  // @ts-ignore
+  ws.sendMsg = (msg: any) => ws.send(JSON.stringify(msg))
+  // @ts-ignore
+  ws.sendData = async (data: any) => {
+    // Wait until the WS is ready to send another message
+    uprogress.status = "uploading"
     await new Promise(resolve => {
       const t = setInterval(() => {
-        if (ws.bufferedAmount > limit) return
+        if (ws.bufferedAmount > 1<<20) return
         resolve(undefined)
         clearInterval(t)
       }, 1)
     })
+    uprogress.status = "processing"
+    ws.send(data)
   }
-  const loc = props.path!.join('/')
-  const files = Array.from(target.files)
-  // Progress stats
-  resetProgress()
-  for (const file of files) {
-    uprogress.filestart.push(uprogress.total)
-    uprogress.total += file.size
-  }
-  console.log(uprogress)
-  // Upload files
-  for (const file of files) {
-    let pos = 0
-    uprogress.filename = file.name
-    uprogress.filesize = file.size
-    uprogress.filepos = 0
-    const name = loc + '/' + (file.webkitRelativePath || file.name)
-    console.log('Uploading', name, file.size)
-    while (pos < file.size) {
-      const end = Math.min(file.size, pos + (1<<20))
-      const value = file.slice(pos, end)
-      ws.send(
-        JSON.stringify({
-          name,
-          size: file.size,
-          start: pos,
-          end,
-        })
-      )
-      ws.send(value)
-      // Wait until the WebSocket is ready to send the next message
-      await wsFlush(1<<20)
-      pos = end
+})
+const worker = async () => {
+  const ws = await WSCreate()
+  while (upqueue.length) {
+    const f = upqueue[0]
+    if (f.cloudPos === f.size) {
+      upqueue.shift()
+      continue
     }
-    await wsFlush(0)
-    console.log('Uploaded', name, pos)
-    ++uprogress.fileidx
+    // Note: files may get modified while we're uploading, f is ours
+    const start = f.cloudPos
+    const end = Math.min(f.size, start + (1<<20))
+    const control = { name: f.cloudName, size: f.size, start, end }
+    const data = f.slice(start, end)
+    ws.sendMsg(control)
+    await ws.sendData(data)
+    f.cloudPos = end
   }
-  resetProgress()
+  if (upqueue.length) startWorker()
+  uprogress.status = "idle"
 }
+let workerRunning: any = false
+const startWorker = () => {
+  if (workerRunning === false) workerRunning = setTimeout(() => {
+    workerRunning = true
+    worker()
+  }, 0)
+}
+
+onMounted(() => {
+  // Need to prevent both to prevent browser from opening the file
+  addEventListener('dragover', uploadHandler)
+  addEventListener('drop', uploadHandler)
+})
+onUnmounted(() => {
+  removeEventListener('dragover', uploadHandler)
+  removeEventListener('drop', uploadHandler)
+})
 </script>
 <template>
   <template>
-    <input
-      ref="fileUploadButton"
-      @change="uploadHandler"
-      class="upload-input"
-      type="file"
-      multiple
-    />
-    <input
-      ref="folderUploadButton"
-      @change="uploadHandler"
-      class="upload-input"
-      type="file"
-      webkitdirectory
-    />
+    <input ref="fileInput" @change="uploadHandler" type="file" multiple>
+    <input ref="folderInput" @change="uploadHandler" type="file" webkitdirectory>
   </template>
-  <SvgButton name="add-file" data-tooltip="Upload files" @click="fileUploadButton.click()" />
-  <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderUploadButton.click()" />
-  <div class="uploadprogress" v-if="uprogress.total">
-    <p>
-      <span v-if="uprogress.filestart.length > 1" class="index">
-        [{{ 1 + uprogress.fileidx }}/{{ uprogress.filestart.length }}]
-      </span>
-      <span class="filename">{{ uprogress.filename }}
-        <span v-if="uprogress.filesize > 1e7 && uprogress.filestart.length > 1" class="percent">
-        {{ (uprogress.filepos / uprogress.filesize * 100).toFixed(0) }} %
+  <SvgButton name="add-file" data-tooltip="Upload files" @click="fileInput.click()" />
+  <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderInput.click()" />
+  <div class="uploadprogress" v-if="uprogress.total" :style="`background: linear-gradient(to right, var(--bar) 0, var(--bar) ${percent}%, var(--nobar) ${percent}%, var(--nobar) 100%);`">
+    <div class="statustext">
+      <span v-if="uprogress.filecount > 1" class="index">
+        [{{ uprogress.fileidx }}/{{ uprogress.filecount }}]
       </span>
+      <span class="filename">{{ uprogress.filename.split('/').pop() }}
+        <span v-if="uprogress.filesize > 1e7" class="percent">
+          {{ (uprogress.filepos / uprogress.filesize * 100).toFixed(0) + '\u202F%' }}
+        </span>
       </span>
       <span class="position" v-if="uprogress.filesize > 1e7">
-        {{ (uprogress.uploaded / 1e6).toFixed(0) }} / {{ (uprogress.total / 1e6).toFixed(0) }} MB
+        {{ (uprogress.uploaded / 1e6).toFixed(0) + '\u202F/\u202F' + (uprogress.total / 1e6).toFixed(0) + '\u202FMB' }}
       </span>
-      <span class="speed">{{ speed.toFixed(speed < 100 ? 1 : 0) }} MB/s</span>
-    </p>
-    <progress :value="uprogress.uploaded" :max="uprogress.total"></progress>
+      <span class="speed">{{ speeddisp }}</span>
+      <button class="close" @click="cancelUploads">❌</button>
+    </div>
   </div>
 </template>
 
 <style scoped>
 .uploadprogress {
-  background: #8888;
+  --bar: var(--accent-color);
+  --nobar: var(--header-background);
   display: flex;
   flex-direction: column;
   color: var(--primary-color);
@@ -158,16 +227,15 @@ async function uploadHandler(event: Event) {
   bottom: 0;
   width: 100vw;
 }
-.uploadprogress p {
+.statustext {
   display: flex;
-  width: 100vw;
-  margin: 0;
+  padding: 0.5rem 0;
 }
 span {
   color: #ccc;
-  margin: 0 0.5em;
   white-space: nowrap;
   text-align: right;
+  padding: 0 0.5em;
 }
 .filename {
   color: #fff;
@@ -177,17 +245,7 @@ span {
   text-overflow: ellipsis;
   text-align: left;
 }
-.index {
-  min-width: 3.5em;
-}
-.position {
-  min-width: 4em;
-}
-.speed {
-  min-width: 4em;
-}
-progress {
-  appearance: progress-bar;
-  width: 100%;
-}
+.index { min-width: 3.5em }
+.position { min-width: 4em }
+.speed { min-width: 4em }
 </style>