diff --git a/frontend/src/components/UploadButton.vue b/frontend/src/components/UploadButton.vue index 68c90af..cce59e3 100644 --- a/frontend/src/components/UploadButton.vue +++ b/frontend/src/components/UploadButton.vue @@ -131,6 +131,7 @@ const uploadCloudFiles = (files: CloudFile[]) => { const cancelUploads = () => { upqueue = [] + blockQueue = [] statReset() } @@ -162,16 +163,42 @@ setInterval(() => { store.uprogress.statdur *= .9 } }, 100) +// Track uploaded bytes for each file to handle out-of-order uploads +const uploadedBytes = new Map>() + const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => { if (name !== store.uprogress.filename) return // If stats have been reset const now = Date.now() - store.uprogress.xfer = store.uprogress.filestart + end - store.uprogress.filepos = end + + // Track which bytes have been uploaded (using start to end range) + if (!uploadedBytes.has(name)) uploadedBytes.set(name, new Set()) + const uploaded = uploadedBytes.get(name)! + const blockSize = 1 << 20 + + // Mark all bytes in this block as uploaded + for (let i = start; i < end; i += blockSize) { + uploaded.add(i) + } + + // Calculate total uploaded bytes for progress + let totalUploaded = 0 + for (let i = 0; i < size; i += blockSize) { + if (uploaded.has(i)) totalUploaded += blockSize + } + + store.uprogress.xfer = store.uprogress.filestart + totalUploaded + store.uprogress.filepos = totalUploaded store.uprogress.statbytes += end - start store.uprogress.statdur += now - store.uprogress.tlast store.uprogress.tlast = now - // File finished? - if (end === size) { + + // Check if file is fully uploaded by examining the block queue + const currentUpload = blockQueue[0] + if (!currentUpload) return + + if (currentUpload.file.cloudName === name && currentUpload.blockIndex >= currentUpload.blocks.length) { + // All blocks for this file have been uploaded + uploadedBytes.delete(name) // Clean up tracking store.uprogress.filestart += size statNextFile() if (++store.uprogress.fileidx >= store.uprogress.filecount) statReset() @@ -198,6 +225,42 @@ const statsAdd = (f: CloudFile[]) => { } let upqueue = [] as CloudFile[] +// Helper function to get upload blocks for a file, prioritizing final 4 blocks if file >= 32 MiB +const getUploadBlocks = (file: CloudFile): {start: number, end: number}[] => { + const BLOCK_SIZE = 1 << 20 // 1 MiB + const MIN_SIZE_FOR_REORDER = 32 * BLOCK_SIZE // 32 MiB = 33554432 bytes + const FINAL_BLOCKS_COUNT = 2 + + const fileSize = file.file.size + const blocks: {start: number, end: number}[] = [] + + if (fileSize >= MIN_SIZE_FOR_REORDER) { + // File is large enough, prioritize final blocks + const finalBlocksStart = fileSize - (FINAL_BLOCKS_COUNT * BLOCK_SIZE) + + // Add final blocks first + for (let i = 0; i < FINAL_BLOCKS_COUNT; i++) { + const start = finalBlocksStart + (i * BLOCK_SIZE) + const end = Math.min(start + BLOCK_SIZE, fileSize) + blocks.push({start, end}) + } + + // Add remaining blocks from beginning + for (let start = 0; start < finalBlocksStart; start += BLOCK_SIZE) { + const end = Math.min(start + BLOCK_SIZE, finalBlocksStart) + blocks.push({start, end}) + } + } else { + // File is smaller, use sequential upload + for (let start = 0; start < fileSize; start += BLOCK_SIZE) { + const end = Math.min(start + BLOCK_SIZE, fileSize) + blocks.push({start, end}) + } + } + + return blocks +} + // TODO: Rewrite as WebSocket class const WSCreate = async () => await new Promise(resolve => { const ws = connect(uploadUrl, { @@ -235,31 +298,58 @@ const WSCreate = async () => await new Promise(resolve => { ws.send(data) } }) + +type BlockUpload = { + file: CloudFile + blocks: {start: number, end: number}[] + blockIndex: number +} + +let blockQueue = [] as BlockUpload[] + const worker = async () => { const ws = await WSCreate() - while (upqueue.length) { - const f = upqueue[0]! - const start = f.cloudPos - const end = Math.min(f.file.size, start + (1<<20)) - const control = { name: f.cloudName, size: f.file.size, start, end } - const data = f.file.slice(start, end) - f.cloudPos = end + while (blockQueue.length) { + const upload = blockQueue[0]! + const f = upload.file + const block = upload.blocks[upload.blockIndex]! + + const control = { name: f.cloudName, size: f.file.size, start: block.start, end: block.end } + const data = f.file.slice(block.start, block.end) + // Note: files may get modified during I/O // @ts-ignore FIXME proper WebSocket class, avoid attaching functions to WebSocket object ws.sendMsg(control) // @ts-ignore await ws.sendData(data) - if (f.cloudPos === f.file.size) upqueue.shift() + + // Move to next block + upload.blockIndex++ + if (upload.blockIndex >= upload.blocks.length) { + // File upload complete + blockQueue.shift() + } } - if (upqueue.length) startWorker() + if (blockQueue.length) startWorker() store.uprogress.status = "idle" workerRunning = false } let workerRunning: any = false const startWorker = () => { if (workerRunning === false) workerRunning = setTimeout(() => { - workerRunning = true - worker() + // Convert new CloudFile entries to BlockUpload entries + while (upqueue.length) { + const file = upqueue.shift()! + const blocks = getUploadBlocks(file) + blockQueue.push({ file, blocks, blockIndex: 0 }) + } + + if (blockQueue.length) { + workerRunning = true + worker() + } else { + workerRunning = false + } }, 0) }