Implement uploads

cista-front/src/App.vue

@@ -1,7 +1,7 @@
 <template>
   <LoginModal />
   <header>
-    <HeaderMain ref="headerMain">
+    <HeaderMain ref="headerMain" :path="path.pathList">
       <HeaderSelected :path="path.pathList" />
     </HeaderMain>
     <BreadCrumb :path="path.pathList" tabindex="-1"/>

cista-front/src/components/HeaderMain.vue

@@ -5,7 +5,7 @@
         <div class="error-message" @click="documentStore.error = ''">{{ documentStore.error }}</div>
         <div class="smallgap"></div>
       </template>
-      <UploadButton />
+      <UploadButton :path="props.path" />
       <SvgButton
         name="create-folder"
         data-tooltip="New folder"
@@ -69,6 +69,9 @@ const settingsMenu = (e: Event) => {
     items,
   })
 }
+const props = defineProps({
+  path: Array<string>
+})
 
 defineExpose({
   toggleSearchInput,

cista-front/src/components/UploadButton.vue

@@ -1,82 +1,112 @@
 <script setup lang="ts">
+import { connect, uploadUrl } from '@/repositories/WS';
 import { useDocumentStore } from '@/stores/documents'
-import { h, ref } from 'vue'
+import { computed, reactive, ref } from 'vue'
 
 const fileUploadButton = ref()
 const folderUploadButton = ref()
 const documentStore = useDocumentStore()
-const open = (placement: any) => openNotification(placement)
-
 const isNotificationOpen = ref(false)
-const openNotification = (placement: any) => {
+const props = defineProps({
-  if (!isNotificationOpen.value) {
+  path: Array<string>
-    /*
-    api.open({
-      message: `Uploading documents`,
-      description: h(NotificationLoading),
-      placement,
-      duration: 0,
-      onClose: () => { isNotificationOpen.value = false }
-    });*/
-    isNotificationOpen.value = true
-  }
-}
-
-function uploadFileHandler() {
-  fileUploadButton.value.click()
-}
-
-async function load(file: File, start: number, end: number): Promise<ArrayBuffer> {
-  const reader = new FileReader()
-  const load = new Promise<Event>(resolve => (reader.onload = resolve))
-  reader.readAsArrayBuffer(file.slice(start, end))
-  const event = await load
-  if (event.target && event.target instanceof FileReader) {
-    return event.target.result as ArrayBuffer
-  } else {
-    throw new Error('Error loading file')
-  }
-}
-
-async function sendChunk(file: File, start: number, end: number) {
-  const ws = documentStore.wsUpload
-  if (ws) {
-    const chunk = await load(file, start, end)
-
-    ws.send(
-      JSON.stringify({
-        name: file.name,
-        size: file.size,
-        start: start,
-        end: end
 })
-    )
-    ws.send(chunk)
-  }
-}
 
+const uprogress_init = {
+  total: 0,
+  uploaded: 0,
+  t0: 0,
+  tlast: 0,
+  filestart: [] as number[],
+  fileidx: 0,
+  filename: '',
+  filepos: 0,
+  filesize: 0,
+}
+const uprogress = reactive({...uprogress_init})
+const speed = computed(() => uprogress.uploaded / (uprogress.tlast - uprogress.t0) / 1e3)
+const resetProgress = () => {
+  Object.assign(uprogress, uprogress_init)
+  uprogress.t0 = Date.now()
+  uprogress.tlast = uprogress.t0 + 1
+}
 async function uploadHandler(event: Event) {
   const target = event.target as HTMLInputElement
-  const chunkSize = 1 << 20
   if (!target?.files?.length) {
     documentStore.error = 'No files selected'
     return
   }
-  for (const idx in target.files) {
+  const ws = await new Promise<WebSocket>(resolve => {
-    const file = target.files[idx]
+    const ws = connect(uploadUrl, {
-    console.log('Uploading', file)
+      open(ev: Event) { resolve(ws) },
-    const numChunks = Math.ceil(file.size / chunkSize)
+      error(ev: Event) {
-    const document = documentStore.pushUploadingDocuments(file.name)
+        console.error('Upload socket error', ev)
-    open('bottomRight')
+        documentStore.error = 'Upload socket error'
-    for (let i = 0; i < numChunks; i++) {
+      },
-      const start = i * chunkSize
+      message(ev: MessageEvent) {
-      const end = Math.min(file.size, start + chunkSize)
+        const res = JSON.parse(ev!.data)
-      const res = await sendChunk(file, start, end)
+        if ('error' in res) {
-      console.log('progress: ' + (100 * (i + 1)) / numChunks)
+          console.error('Upload socket error', res.error)
-      console.log('Num Chunks: ' + numChunks)
+          documentStore.error = res.error.message
-      documentStore.updateUploadingDocuments(document.key, (100 * (i + 1)) / numChunks)
+          return
         }
+        if (res.status === 'ack') {
+          // Upload progress upgrade on SERVER ACK
+          const fstart = uprogress.filestart[uprogress.fileidx]
+          uprogress.uploaded = fstart + res.req.end
+          uprogress.filepos = res.req.end
+          uprogress.tlast = Date.now()
+        } else console.log('Unknown upload response', res)
+      },
+    })
+    ws.binaryType = 'arraybuffer'
+  })
+  async function wsFlush(limit: number) {
+    await new Promise(resolve => {
+      const t = setInterval(() => {
+        if (ws.bufferedAmount > limit) return
+        resolve(undefined)
+        clearInterval(t)
+      }, 1)
+    })
   }
+  const loc = props.path!.join('/')
+  const files = Array.from(target.files)
+  // Progress stats
+  resetProgress()
+  for (const file of files) {
+    uprogress.filestart.push(uprogress.total)
+    uprogress.total += file.size
+  }
+  console.log(uprogress)
+  // Upload files
+  for (const file of files) {
+    let pos = 0
+    uprogress.filename = file.name
+    uprogress.filesize = file.size
+    uprogress.filepos = 0
+    const name = loc + '/' + (file.webkitRelativePath || file.name)
+    console.log('Uploading', name, file.size)
+    while (pos < file.size) {
+      const end = Math.min(file.size, pos + (1<<20))
+      const value = file.slice(pos, end)
+      ws.send(
+        JSON.stringify({
+          name,
+          size: file.size,
+          start: pos,
+          end,
+        })
+      )
+      ws.send(value)
+      // Wait until the WebSocket is ready to send the next message
+      await wsFlush(1<<20)
+      pos = end
+    }
+    await wsFlush(0)
+    console.log('Uploaded', name, pos)
+    ++uprogress.fileidx
+  }
+  resetProgress()
 }
 </script>
 <template>
@@ -98,4 +128,66 @@ async function uploadHandler(event: Event) {
   </template>
   <SvgButton name="add-file" data-tooltip="Upload files" @click="fileUploadButton.click()" />
   <SvgButton name="add-folder" data-tooltip="Upload folder" @click="folderUploadButton.click()" />
+  <div class="uploadprogress" v-if="uprogress.total">
+    <p>
+      <span v-if="uprogress.filestart.length > 1" class="index">
+        [{{ 1 + uprogress.fileidx }}/{{ uprogress.filestart.length }}]
+      </span>
+      <span class="filename">{{ uprogress.filename }}
+        <span v-if="uprogress.filesize > 1e7 && uprogress.filestart.length > 1" class="percent">
+        {{ (uprogress.filepos / uprogress.filesize * 100).toFixed(0) }} %
+      </span>
+      </span>
+      <span class="position" v-if="uprogress.filesize > 1e7">
+        {{ (uprogress.uploaded / 1e6).toFixed(0) }} / {{ (uprogress.total / 1e6).toFixed(0) }} MB
+      </span>
+      <span class="speed">{{ speed.toFixed(speed < 100 ? 1 : 0) }} MB/s</span>
+    </p>
+    <progress :value="uprogress.uploaded" :max="uprogress.total"></progress>
+  </div>
 </template>
+
+<style scoped>
+.uploadprogress {
+  background: #8888;
+  display: flex;
+  flex-direction: column;
+  color: var(--primary-color);
+  position: fixed;
+  left: 0;
+  bottom: 0;
+  width: 100vw;
+}
+.uploadprogress p {
+  display: flex;
+  width: 100vw;
+  margin: 0;
+}
+span {
+  color: #ccc;
+  margin: 0 0.5em;
+  white-space: nowrap;
+  text-align: right;
+}
+.filename {
+  color: #fff;
+  flex: 1 1;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  text-align: left;
+}
+.index {
+  min-width: 3.5em;
+}
+.position {
+  min-width: 4em;
+}
+.speed {
+  min-width: 4em;
+}
+progress {
+  appearance: progress-bar;
+  width: 100%;
+}
+</style>

cista-front/src/stores/documents.ts

@@ -82,29 +82,6 @@ export const useDocumentStore = defineStore({
       )
       this.document = docs as Document[]
     },
-    updateUploadingDocuments(key: number, progress: number) {
-      for (const d of this.uploadingDocuments) {
-        if (d.key === key) d.progress = progress
-      }
-    },
-    pushUploadingDocuments(name: string) {
-      this.uploadCount++
-      const document = {
-        key: this.uploadCount,
-        name: name,
-        progress: 0
-      }
-      this.uploadingDocuments.push(document)
-      return document
-    },
-    deleteUploadingDocument(key: number) {
-      this.uploadingDocuments = this.uploadingDocuments.filter(e => e.key !== key)
-    },
-    updateModified() {
-      for (const d of this.document) {
-        if ('mtime' in d) d.modified = formatUnixDate(d.mtime)
-      }
-    },
     login(username: string, privileged: boolean) {
       this.user.username = username
       this.user.privileged = privileged

cista/api.py

@@ -46,6 +46,7 @@ async def upload(req, ws):
             raise ValueError(f"Expected {req.end - pos} more bytes, got {d}")
         # Report success
         res = StatusMsg(status="ack", req=req)
+        print("ack", res)
         await asend(ws, res)