Uploads by PUT range requests rather than WS, remove dead code WS handlers #9

Merged
LeoVasanko merged 3 commits from put-uploads into main 2026-04-25 02:55:52 +00:00
6 changed files with 357 additions and 209 deletions
+1 -61
View File
@@ -1,5 +1,4 @@
import asyncio import asyncio
import typing
from pathlib import PurePosixPath from pathlib import PurePosixPath
from secrets import token_bytes from secrets import token_bytes
@@ -9,7 +8,7 @@ from sanic.exceptions import BadRequest
from cista import __version__, auth, config, sso, watching from cista import __version__, auth, config, sso, watching
from cista.fileio import FileServer from cista.fileio import FileServer
from cista.protocol import ControlTypes, FileRange, StatusMsg from cista.protocol import ControlTypes, StatusMsg
from cista.util.apphelpers import asend, websocket_wrapper from cista.util.apphelpers import asend, websocket_wrapper
bp = Blueprint("api", url_prefix="/api") bp = Blueprint("api", url_prefix="/api")
@@ -26,65 +25,6 @@ async def stop_fileserver(app):
await fileserver.stop() await fileserver.stop()
@bp.websocket("upload")
@websocket_wrapper
async def upload(req, ws):
alink = fileserver.alink
while True:
req = None
text = await ws.recv()
if not isinstance(text, str):
raise ValueError(
f"Expected JSON control, got binary len(data) = {len(text)}",
)
req = msgspec.json.decode(text, type=FileRange)
pos = req.start
while True:
data = await ws.recv()
if not isinstance(data, bytes):
break
if len(data) > req.end - pos:
raise ValueError(
f"Expected up to {req.end - pos} bytes, got {len(data)} bytes"
)
sentsize = await alink(("upload", req.name, pos, data, req.size))
pos += typing.cast(int, sentsize)
if pos >= req.end:
break
if pos != req.end:
d = f"{len(data)} bytes" if isinstance(data, bytes) else data
raise ValueError(f"Expected {req.end - pos} more bytes, got {d}")
# Signal the watcher about the uploaded file and its parent directories
path = PurePosixPath(req.name)
watching.notify_change(path, *path.parents)
# Report success
res = StatusMsg(status="ack", req=req)
await asend(ws, res)
@bp.websocket("download")
@websocket_wrapper
async def download(req, ws):
alink = fileserver.alink
while True:
req = None
text = await ws.recv()
if not isinstance(text, str):
raise ValueError(
f"Expected JSON control, got binary len(data) = {len(text)}",
)
req = msgspec.json.decode(text, type=FileRange)
pos = req.start
while pos < req.end:
end = min(req.end, pos + (1 << 20))
data = typing.cast(bytes, await alink(("download", req.name, pos, end)))
await asend(ws, data)
pos += len(data)
# Report success
res = StatusMsg(status="ack", req=req)
await asend(ws, res)
@bp.websocket("control") @bp.websocket("control")
@websocket_wrapper @websocket_wrapper
async def control(req, ws): async def control(req, ws):
+90 -3
View File
@@ -1,6 +1,7 @@
import asyncio import asyncio
import datetime import datetime
import mimetypes import mimetypes
import re
import time import time
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from multiprocessing import cpu_count from multiprocessing import cpu_count
@@ -11,8 +12,8 @@ from wsgiref.handlers import format_date_time
import sanic.helpers import sanic.helpers
from blake3 import blake3 from blake3 import blake3
from sanic import Blueprint, Sanic, empty, raw, redirect from sanic import Blueprint, Sanic, empty, json, raw, redirect
from sanic.exceptions import Forbidden, NotFound from sanic.exceptions import BadRequest, Forbidden, NotFound
from sanic.log import logger from sanic.log import logger
from setproctitle import setproctitle from setproctitle import setproctitle
from stream_zip import ZIP_AUTO, stream_zip from stream_zip import ZIP_AUTO, stream_zip
@@ -20,7 +21,7 @@ from zstandard import ZstdCompressor
from cista import auth, config, preview, session, sso, watching from cista import auth, config, preview, session, sso, watching
from cista.preview import shutdown_preview_workers, start_preview_workers from cista.preview import shutdown_preview_workers, start_preview_workers
from cista.api import bp from cista.api import bp, fileserver
from cista.sanic_logging import configure_access_logging, configure_main_logging, format_access_log from cista.sanic_logging import configure_access_logging, configure_main_logging, format_access_log
from cista.sanic_logging import logger as access_logger from cista.sanic_logging import logger as access_logger
from cista.util.apphelpers import handle_sanic_exception from cista.util.apphelpers import handle_sanic_exception
@@ -127,6 +128,68 @@ def http_fileserver(app):
"""Verify access to file server routes.""" """Verify access to file server routes."""
await auth.verify(request) await auth.verify(request)
@bp.put("/files/<name:path>")
async def upload_file_chunk(request, *args, **kwargs):
body = request.body
header = request.headers.get("content-range")
if header:
start, end, total = _parse_content_range(header, len(body))
else:
start = 0
end = len(body)
total = end
raw_name = kwargs.get("name")
if raw_name is None and args:
raw_name = args[0]
if not isinstance(raw_name, str) or not raw_name:
prefix = "/files/"
if not request.path.startswith(prefix):
raise BadRequest("Invalid upload path")
raw_name = request.path[len(prefix) :]
rel_name = unquote(raw_name)
upload_info = await asyncio.to_thread(
fileserver.upload_info,
rel_name,
start,
body,
total,
)
extras = []
chunk_len = end - start
whole_file = start == 0 and end == total
if not whole_file:
start_mib = _to_mib_int(start)
chunk_mib = _to_mib_int(chunk_len)
# Keep range logs compact for fixed-size upload blocks.
if chunk_mib == 16:
extras.append(f"{start_mib}MiB")
else:
extras.append(f"{start_mib}+{chunk_mib}MiB")
if upload_info.get("created"):
extras.append(f"created {_to_mib_int(total)}MiB")
size_before = upload_info.get("size_before")
size_after = upload_info.get("size_after")
if (
size_before is not None
and size_after is not None
and size_before != size_after
):
extras.append("resized")
request.ctx._log_extra = " ".join(extras) if extras else None
path = PurePosixPath(rel_name)
watching.notify_change(path, *path.parents)
return json(
{
"status": "ack",
"req": {
"name": rel_name,
"size": total,
"start": start,
"end": end,
},
}
)
bp.static( bp.static(
"/files/", "/files/",
config.config.path, config.config.path,
@@ -138,6 +201,30 @@ def http_fileserver(app):
www = {} www = {}
_CONTENT_RANGE_RE = re.compile(r"^bytes (\d+)-(\d+)/(\d+)$")
def _parse_content_range(header: str, body_len: int) -> tuple[int, int, int]:
m = _CONTENT_RANGE_RE.fullmatch(header.strip())
if m is None:
raise BadRequest("Invalid Content-Range format")
start, end_inclusive, total = (int(v) for v in m.groups())
if total <= 0:
raise BadRequest("Invalid Content-Range total size")
if start > end_inclusive:
raise BadRequest("Invalid Content-Range range")
if end_inclusive >= total:
raise BadRequest("Content-Range exceeds total size")
expected_len = end_inclusive - start + 1
if expected_len != body_len:
raise BadRequest(
f"Content length mismatch for range: expected {expected_len}, got {body_len}"
)
return start, end_inclusive + 1, total
def _to_mib_int(value_bytes: int) -> int:
return round(value_bytes / (1 << 20))
def _load_wwwroot(www): def _load_wwwroot(www):
+23 -30
View File
@@ -1,9 +1,8 @@
import asyncio
import os import os
import threading
from cista import config from cista import config
from cista.util import filename from cista.util import filename
from cista.util.asynclink import AsyncLink
from cista.util.lrucache import LRUCache from cista.util.lrucache import LRUCache
@@ -62,38 +61,32 @@ class File:
class FileServer: class FileServer:
async def start(self): async def start(self):
self.alink = AsyncLink()
self.worker = asyncio.get_event_loop().run_in_executor(
None,
self.worker_thread,
self.alink.to_sync,
)
self.cache = LRUCache(File, capacity=10, maxage=5.0) self.cache = LRUCache(File, capacity=10, maxage=5.0)
self.cache_lock = threading.Lock()
self.file_locks: dict[str, threading.Lock] = {}
async def stop(self): async def stop(self):
await self.alink.stop() self.cache.close()
await self.worker
def worker_thread(self, slink): @staticmethod
def _stat_size(path):
try: try:
for req in slink: return os.stat(path).st_size
with req as (command, *args): except FileNotFoundError:
if command == "upload": return None
req.set_result(self.upload(*args))
elif command == "download":
req.set_result(self.download(*args))
else:
raise NotImplementedError(f"Unhandled {command=} {args}")
finally:
self.cache.close()
def upload(self, name, pos, data, file_size): def upload_info(self, name, pos, data, file_size):
name = filename.sanitize(name) name = filename.sanitize(name)
f = self.cache[name] with self.cache_lock:
f.write(pos, data, file_size=file_size) f = self.cache[name]
return len(data) lock = self.file_locks.setdefault(name, threading.Lock())
with lock:
def download(self, name, start, end): size_before = self._stat_size(f.path)
name = filename.sanitize(name) f.write(pos, data, file_size=file_size)
f = self.cache[name] size_after = self._stat_size(f.path)
return f[start:end] return {
"written": len(data),
"created": size_before is None,
"size_before": size_before,
"size_after": size_after,
}
+1 -12
View File
@@ -12,7 +12,6 @@ from cista.util import filename
## Control commands ## Control commands
class ControlBase(msgspec.Struct, tag_field="op", tag=str.lower): class ControlBase(msgspec.Struct, tag_field="op", tag=str.lower):
def __call__(self): def __call__(self):
raise NotImplementedError raise NotImplementedError
@@ -118,19 +117,9 @@ class Cp(ControlBase):
ControlTypes = MkDir | Rename | Rm | Mv | Cp ControlTypes = MkDir | Rename | Rm | Mv | Cp
## File uploads and downloads
class FileRange(msgspec.Struct):
name: str
size: int
start: int
end: int
class StatusMsg(msgspec.Struct): class StatusMsg(msgspec.Struct):
status: str status: str
req: FileRange req: Any
class ErrorMsg(msgspec.Struct): class ErrorMsg(msgspec.Struct):
+242 -102
View File
@@ -8,12 +8,11 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import { connect, uploadUrl } from '@/repositories/WS';
import { useMainStore } from '@/stores/main' import { useMainStore } from '@/stores/main'
import { getDocuments } from '@/stores/documentStore' import { getDocuments } from '@/stores/documentStore'
import { Doc } from '@/repositories/Document' import { Doc } from '@/repositories/Document'
import { collator } from '@/utils'; import { collator } from '@/utils';
import { onMounted, onUnmounted, reactive, ref } from 'vue' import { onMounted, onUnmounted, ref } from 'vue'
import { useRouter } from 'vue-router' import { useRouter } from 'vue-router'
const router = useRouter() const router = useRouter()
@@ -29,6 +28,22 @@ type CloudFile = {
cloudName: string cloudName: string
cloudPos: number cloudPos: number
} }
type UploadRange = {
name: string
size: number
start: number
end: number
}
type InflightBlock = {
name: string
start: number
end: number
startedAt: number
}
const UPLOAD_BLOCK_SIZE = 16 << 20 // 16 MiB
function pasteHandler(event: ClipboardEvent) { function pasteHandler(event: ClipboardEvent) {
const items = Array.from(event.clipboardData?.items ?? []) const items = Array.from(event.clipboardData?.items ?? [])
const infiles = [] as File[] const infiles = [] as File[]
@@ -46,7 +61,8 @@ function pasteHandler(event: ClipboardEvent) {
if (infiles.length || dirs.length) { if (infiles.length || dirs.length) {
event.preventDefault() event.preventDefault()
uploadFiles(infiles) uploadFiles(infiles)
for (const entry of dirs) pasteDirectory(entry, `${props.path!.join('/')}/${entry.name}`) const base = props.path!.join('/')
for (const entry of dirs) pasteDirectory(entry, `${base ? `${base}/` : ''}${entry.name}`)
} }
} }
const pasteDirectory = async (entry: FileSystemDirectoryEntry, loc: string) => { const pasteDirectory = async (entry: FileSystemDirectoryEntry, loc: string) => {
@@ -82,7 +98,7 @@ const uploadFiles = (infiles: File[]) => {
if (!folderName && file.webkitRelativePath) folderName = relPath.split('/')[0] ?? '' if (!folderName && file.webkitRelativePath) folderName = relPath.split('/')[0] ?? ''
files.push({ files.push({
file, file,
cloudName: loc + '/' + relPath, cloudName: `${loc ? `${loc}/` : ''}${relPath}`,
cloudPos: 0, cloudPos: 0,
}) })
} }
@@ -130,8 +146,12 @@ const uploadCloudFiles = (files: CloudFile[]) => {
} }
const cancelUploads = () => { const cancelUploads = () => {
uploadRunId += 1
upqueue = [] upqueue = []
blockQueue = [] blockQueue = []
inflightBlocks.clear()
uploadedBytes.clear()
store.uprogress.status = 'idle'
statReset() statReset()
} }
@@ -152,51 +172,98 @@ const uprogress_init = {
status: 'idle', status: 'idle',
} }
store.uprogress = {...uprogress_init} store.uprogress = {...uprogress_init}
// Track uploaded bytes for each file to handle out-of-order uploads
const uploadedBytes = new Map<string, Set<number>>()
const inflightBlocks = new Map<string, InflightBlock>()
let smoothedBlockMs = 1500
let lastProgressTick = Date.now()
let lastVisualUploaded = 0
const inflightKey = (name: string, start: number) => `${name}:${start}`
const completedUploadedBytes = (name: string, size: number) => {
const uploaded = uploadedBytes.get(name)
if (!uploaded) return 0
const blockSize = UPLOAD_BLOCK_SIZE
let total = 0
for (let i = 0; i < size; i += blockSize) {
if (uploaded.has(i)) total += Math.min(blockSize, size - i)
}
return total
}
const simulatedInflightBytes = (name: string, now: number) => {
let total = 0
for (const block of inflightBlocks.values()) {
if (block.name !== name) continue
const size = block.end - block.start
const elapsed = Math.max(0, now - block.startedAt)
const fraction = Math.min(0.98, elapsed / Math.max(200, smoothedBlockMs))
total += size * fraction
}
return total
}
const refreshProgress = (now: number) => {
const name = store.uprogress.filename
const size = store.uprogress.filesize
if (!name || !size) {
lastProgressTick = now
return 0
}
const completed = completedUploadedBytes(name, size)
const estimated = simulatedInflightBytes(name, now)
const visualUploaded = Math.min(size, Math.round(completed + estimated))
const delta = Math.max(0, visualUploaded - lastVisualUploaded)
const dt = Math.max(1, now - lastProgressTick)
store.uprogress.filepos = visualUploaded
store.uprogress.xfer = store.uprogress.filestart + visualUploaded
if (delta > 0) {
store.uprogress.statbytes += delta
store.uprogress.statdur += dt
store.uprogress.tlast = now
}
lastVisualUploaded = visualUploaded
lastProgressTick = now
return delta
}
setInterval(() => { setInterval(() => {
if (Date.now() - store.uprogress.tlast > 3000) { const now = Date.now()
// Reset const delta = refreshProgress(now)
if (delta > 0) return
if (now - store.uprogress.tlast > 3000) {
store.uprogress.statbytes = 0 store.uprogress.statbytes = 0
store.uprogress.statdur = 1 store.uprogress.statdur = 1
} else { } else {
// Running average by decay store.uprogress.statbytes *= .95
store.uprogress.statbytes *= .9 store.uprogress.statdur *= .95
store.uprogress.statdur *= .9
} }
}, 100) }, 100)
// Track uploaded bytes for each file to handle out-of-order uploads
const uploadedBytes = new Map<string, Set<number>>()
const statUpdate = ({name, size, start, end}: {name: string, size: number, start: number, end: number}) => { const statUpdate = ({name, size, start, end}: UploadRange) => {
if (name !== store.uprogress.filename) return // If stats have been reset if (name !== store.uprogress.filename) return // If stats have been reset
const now = Date.now()
// Track which bytes have been uploaded (using start to end range) // Track which bytes have been uploaded (using start to end range)
if (!uploadedBytes.has(name)) uploadedBytes.set(name, new Set()) if (!uploadedBytes.has(name)) uploadedBytes.set(name, new Set())
const uploaded = uploadedBytes.get(name)! const uploaded = uploadedBytes.get(name)!
const blockSize = 1 << 20 const blockSize = UPLOAD_BLOCK_SIZE
// Mark all bytes in this block as uploaded // Mark all bytes in this block as uploaded
for (let i = start; i < end; i += blockSize) { for (let i = start; i < end; i += blockSize) {
uploaded.add(i) uploaded.add(i)
} }
refreshProgress(Date.now())
// Calculate total uploaded bytes for progress
let totalUploaded = 0
for (let i = 0; i < size; i += blockSize) {
if (uploaded.has(i)) totalUploaded += blockSize
}
store.uprogress.xfer = store.uprogress.filestart + totalUploaded
store.uprogress.filepos = totalUploaded
store.uprogress.statbytes += end - start
store.uprogress.statdur += now - store.uprogress.tlast
store.uprogress.tlast = now
// Check if file is fully uploaded by examining the block queue // Check if file is fully uploaded by examining the block queue
const currentUpload = blockQueue[0] const currentUpload = blockQueue[0]
if (!currentUpload) return if (!currentUpload) return
if (currentUpload.file.cloudName === name && currentUpload.blockIndex >= currentUpload.blocks.length) { if (currentUpload.file.cloudName === name && currentUpload.completed >= currentUpload.blocks.length) {
// All blocks for this file have been uploaded // All blocks for this file have been uploaded
uploadedBytes.delete(name) // Clean up tracking uploadedBytes.delete(name) // Clean up tracking
store.uprogress.filestart += size store.uprogress.filestart += size
@@ -210,11 +277,15 @@ const statNextFile = () => {
store.uprogress.filepos = 0 store.uprogress.filepos = 0
store.uprogress.filesize = f.file.size store.uprogress.filesize = f.file.size
store.uprogress.filename = f.cloudName store.uprogress.filename = f.cloudName
lastVisualUploaded = 0
lastProgressTick = Date.now()
} }
const statReset = () => { const statReset = () => {
Object.assign(store.uprogress, uprogress_init) Object.assign(store.uprogress, uprogress_init)
store.uprogress.t0 = Date.now() store.uprogress.t0 = Date.now()
store.uprogress.tlast = store.uprogress.t0 + 1 store.uprogress.tlast = store.uprogress.t0 + 1
lastVisualUploaded = 0
lastProgressTick = store.uprogress.t0
} }
const statsAdd = (f: CloudFile[]) => { const statsAdd = (f: CloudFile[]) => {
if (store.uprogress.files.length === 0) statReset() if (store.uprogress.files.length === 0) statReset()
@@ -224,10 +295,12 @@ const statsAdd = (f: CloudFile[]) => {
statNextFile() statNextFile()
} }
let upqueue = [] as CloudFile[] let upqueue = [] as CloudFile[]
const MAX_PARALLEL_REQUESTS = 4
const RETRY_DELAY_MS = 400
// Helper function to get upload blocks for a file, prioritizing final 4 blocks if file >= 32 MiB // Helper function to get upload blocks for a file, prioritizing final 4 blocks if file >= 32 MiB
const getUploadBlocks = (file: CloudFile): {start: number, end: number}[] => { const getUploadBlocks = (file: CloudFile): {start: number, end: number}[] => {
const BLOCK_SIZE = 1 << 20 // 1 MiB const BLOCK_SIZE = UPLOAD_BLOCK_SIZE
const MIN_SIZE_FOR_REORDER = 32 * BLOCK_SIZE // 32 MiB = 33554432 bytes const MIN_SIZE_FOR_REORDER = 32 * BLOCK_SIZE // 32 MiB = 33554432 bytes
const FINAL_BLOCKS_COUNT = 2 const FINAL_BLOCKS_COUNT = 2
@@ -261,95 +334,162 @@ const getUploadBlocks = (file: CloudFile): {start: number, end: number}[] => {
return blocks return blocks
} }
// TODO: Rewrite as WebSocket class
const WSCreate = async () => await new Promise<WebSocket>(resolve => {
const ws = connect(uploadUrl, {
open(ev: Event) { resolve(ws) },
error(ev: Event) {
console.error('Upload socket error', ev)
store.error = 'Upload socket error'
},
message(ev: MessageEvent) {
const res = JSON.parse(ev!.data)
if ('error' in res) {
console.error('Upload socket error', res.error)
store.error = res.error.message
return
}
if (res.status === 'ack') {
statUpdate(res.req)
} else console.log('Unknown upload response', res)
},
})
// @ts-ignore
ws.sendMsg = (msg: any) => ws.send(JSON.stringify(msg))
// @ts-ignore
ws.sendData = async (data: any) => {
// Wait until the WS is ready to send another message
store.uprogress.status = "uploading"
await new Promise(resolve => {
const t = setInterval(() => {
if (ws.bufferedAmount > 1<<20) return
resolve(undefined)
clearInterval(t)
}, 1)
})
store.uprogress.status = "processing"
ws.send(data)
}
})
type BlockUpload = { type BlockUpload = {
file: CloudFile file: CloudFile
blocks: {start: number, end: number}[] blocks: {start: number, end: number}[]
blockIndex: number nextIndex: number
completed: number
runId: number
} }
let blockQueue = [] as BlockUpload[] let blockQueue = [] as BlockUpload[]
let workerRunning = false
let uploadRunId = 0
const worker = async () => { const enqueuePendingUploads = () => {
const ws = await WSCreate() while (upqueue.length) {
while (blockQueue.length) { const file = upqueue.shift()!
const upload = blockQueue[0]! const blocks = getUploadBlocks(file)
const f = upload.file blockQueue.push({ file, blocks, nextIndex: 0, completed: 0, runId: uploadRunId })
const block = upload.blocks[upload.blockIndex]! }
}
const control = { name: f.cloudName, size: f.file.size, start: block.start, end: block.end } const uploadUrlForFile = (cloudName: string) => {
const data = f.file.slice(block.start, block.end) const normalized = cloudName.replace(/^\/+/, '')
const encoded = normalized.split('/').map(encodeURIComponent).join('/')
return `/files/${encoded}`
}
// Note: files may get modified during I/O const uploadBlock = async (upload: BlockUpload, block: {start: number, end: number}) => {
// @ts-ignore FIXME proper WebSocket class, avoid attaching functions to WebSocket object const body = upload.file.file.slice(block.start, block.end)
ws.sendMsg(control) const range = `bytes ${block.start}-${block.end - 1}/${upload.file.file.size}`
// @ts-ignore const fallbackReq = {
await ws.sendData(data) name: upload.file.cloudName,
size: upload.file.file.size,
start: block.start,
end: block.end,
}
let attempt = 0
// Move to next block while (true) {
upload.blockIndex++ attempt += 1
if (upload.blockIndex >= upload.blocks.length) { if (upload.runId !== uploadRunId) throw new Error('Upload cancelled')
// File upload complete try {
blockQueue.shift() const res = await fetch(uploadUrlForFile(upload.file.cloudName), {
method: 'PUT',
headers: {
'Content-Type': 'application/octet-stream',
'Content-Range': range,
},
body,
})
if (!res.ok) {
const message = await res.text().catch(() => '')
const retryable = res.status >= 500 || res.status === 408 || res.status === 429
if (!retryable) throw new Error(message || `HTTP ${res.status}`)
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS))
continue
}
const payload = await res.json().catch(() => null)
return payload?.status === 'ack' && payload.req ? payload.req : fallbackReq
} catch (err: any) {
const message = err instanceof Error ? err.message : String(err)
if (message === 'Upload cancelled') throw err
if (upload.runId !== uploadRunId) throw new Error('Upload cancelled')
if (attempt % 10 === 0) {
console.warn(`Upload retry ${attempt} for ${upload.file.cloudName}: ${message}`)
}
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS))
} }
} }
if (blockQueue.length) startWorker()
store.uprogress.status = "idle"
workerRunning = false
} }
let workerRunning: any = false
const startWorker = () => {
if (workerRunning === false) workerRunning = setTimeout(() => {
// Convert new CloudFile entries to BlockUpload entries
while (upqueue.length) {
const file = upqueue.shift()!
const blocks = getUploadBlocks(file)
blockQueue.push({ file, blocks, blockIndex: 0 })
}
if (blockQueue.length) { const startInflightBlock = (name: string, block: {start: number, end: number}) => {
workerRunning = true inflightBlocks.set(inflightKey(name, block.start), {
worker() name,
} else { start: block.start,
workerRunning = false end: block.end,
startedAt: Date.now(),
})
}
const finishInflightBlock = (name: string, block: {start: number, end: number}) => {
const key = inflightKey(name, block.start)
const info = inflightBlocks.get(key)
if (!info) return
const elapsed = Math.max(1, Date.now() - info.startedAt)
smoothedBlockMs = smoothedBlockMs * 0.85 + elapsed * 0.15
inflightBlocks.delete(key)
}
const worker = async (runId: number) => {
try {
while (runId === uploadRunId) {
enqueuePendingUploads()
if (!blockQueue.length) break
const upload = blockQueue[0]!
const inflight = new Set<Promise<void>>()
while (runId === uploadRunId && upload.completed < upload.blocks.length) {
while (
runId === uploadRunId
&& upload.nextIndex < upload.blocks.length
&& inflight.size < MAX_PARALLEL_REQUESTS
) {
const block = upload.blocks[upload.nextIndex++]!
store.uprogress.status = 'uploading'
startInflightBlock(upload.file.cloudName, block)
let task: Promise<void>
task = uploadBlock(upload, block)
.then(req => {
finishInflightBlock(upload.file.cloudName, block)
upload.completed += 1
statUpdate(req)
})
.catch(err => {
finishInflightBlock(upload.file.cloudName, block)
throw err
})
.finally(() => {
inflight.delete(task)
})
inflight.add(task)
}
if (!inflight.size) break
await Promise.race(inflight)
}
if (runId !== uploadRunId) return
if (upload.completed >= upload.blocks.length) {
blockQueue.shift()
} else {
break
}
} }
} catch (err: any) {
if (runId !== uploadRunId) return
console.error('Upload error', err)
store.error = err?.message || 'Upload failed'
uploadRunId += 1
upqueue = []
blockQueue = []
inflightBlocks.clear()
} finally {
store.uprogress.status = 'idle'
workerRunning = false
if (upqueue.length) startWorker()
}
}
const startWorker = () => {
if (workerRunning) return
workerRunning = true
const runId = uploadRunId
setTimeout(() => {
void worker(runId)
}, 0) }, 0)
} }
-1
View File
@@ -3,7 +3,6 @@ import { showAuthIframe, AuthCancelledError, isAuthIframeOpen } from 'paskia'
import type { FileEntry, UpdateEntry, errorEvent } from "./Document" import type { FileEntry, UpdateEntry, errorEvent } from "./Document"
export const controlUrl = '/api/control' export const controlUrl = '/api/control'
export const uploadUrl = '/api/upload'
export const watchUrl = '/api/watch' export const watchUrl = '/api/watch'
let tree = [] as FileEntry[] let tree = [] as FileEntry[]