Better special handling for root Document. Restore live updates of relative modified times.

README.md

@@ -1,6 +1,7 @@
 # Web File Storage
 
 Run directly from repository with Hatch (or use pip install as usual):
+
 ```sh
 hatch run cista -l :3000 /path/to/files
 ```
@@ -8,16 +9,17 @@ hatch run cista -l :3000 /path/to/files
 Settings incl. these arguments are stored to config file on the first startup and later `hatch run cista` is sufficient. If the `cista` script is missing, consider `pip install -e .` (within `hatch shell`) or some other trickery (known issue with installs made prior to adding the startup script).
 
 Create your user account:
+
 ```sh
 hatch run cista --user admin --privileged
 ```
 
 ## Build frontend
 
-Prebuilt frontend is provided in repository but for any changes it will need to be manually rebuilt:
+Frontend needs to be built before using and after any frontend changes:
 
 ```sh
-cd cista-front
+cd frontend
 npm install
 npm run build
 ```

cista/api.py

@@ -104,11 +104,11 @@ async def watch(req, ws):
     )
     uuid = token_bytes(16)
     try:
-        with watching.tree_lock:
+        with watching.state.lock:
             q = watching.pubsub[uuid] = asyncio.Queue()
             # Init with disk usage and full tree
-            await ws.send(watching.format_du())
-            await ws.send(watching.format_tree())
+            await ws.send(watching.format_space(watching.state.space))
+            await ws.send(watching.format_root(watching.state.root))
         # Send updates
         while True:
             await ws.send(await q.get())

cista/app.py

@@ -1,10 +1,8 @@
 import asyncio
 import datetime
 import mimetypes
-from collections import deque
 from concurrent.futures import ThreadPoolExecutor
-from importlib.resources import files
-from pathlib import Path
+from pathlib import Path, PurePath, PurePosixPath
 from stat import S_IFDIR, S_IFREG
 from urllib.parse import unquote
 from wsgiref.handlers import format_date_time
@@ -12,15 +10,13 @@ from wsgiref.handlers import format_date_time
 import brotli
 import sanic.helpers
 from blake3 import blake3
-from natsort import natsorted, ns
 from sanic import Blueprint, Sanic, empty, raw
-from sanic.exceptions import Forbidden, NotFound
+from sanic.exceptions import Forbidden, NotFound, ServerError
 from sanic.log import logging
 from stream_zip import ZIP_AUTO, stream_zip
 
 from cista import auth, config, session, watching
 from cista.api import bp
-from cista.protocol import DirEntry
 from cista.util.apphelpers import handle_sanic_exception
 
 # Workaround until Sanic PR #2824 is merged
@@ -36,7 +32,9 @@ app.exception(Exception)(handle_sanic_exception)
 async def main_start(app, loop):
     config.load_config()
     await watching.start(app, loop)
-    app.ctx.threadexec = ThreadPoolExecutor(max_workers=8)
+    app.ctx.threadexec = ThreadPoolExecutor(
+        max_workers=8, thread_name_prefix="cista-ioworker"
+    )
 
 
 @app.after_server_stop
@@ -49,8 +47,8 @@ async def main_stop(app, loop):
 async def use_session(req):
     req.ctx.session = session.get(req)
     try:
-        req.ctx.username = req.ctx.session["username"]
-        req.ctx.user = config.config.users[req.ctx.session["username"]]  # type: ignore
+        req.ctx.username = req.ctx.session["username"]  # type: ignore
+        req.ctx.user = config.config.users[req.ctx.username]
     except (AttributeError, KeyError, TypeError):
         req.ctx.username = None
         req.ctx.user = None
@@ -81,22 +79,16 @@ def http_fileserver(app, _):
 www = {}
 
 
-@app.before_server_start
-async def load_wwwroot(*_ignored):
-    global www
-    www = await asyncio.get_event_loop().run_in_executor(None, _load_wwwroot, www)
-
-
 def _load_wwwroot(www):
     wwwnew = {}
-    base = files("cista") / "wwwroot"
-    paths = ["."]
+    base = Path(__file__).with_name("wwwroot")
+    paths = [PurePath()]
     while paths:
         path = paths.pop(0)
         current = base / path
         for p in current.iterdir():
             if p.is_dir():
-                paths.append(current / p.parts[-1])
+                paths.append(p.relative_to(base))
                 continue
             name = p.relative_to(base).as_posix()
             mime = mimetypes.guess_type(name)[0] or "application/octet-stream"
@@ -127,15 +119,35 @@ def _load_wwwroot(www):
             if len(br) >= len(data):
                 br = False
             wwwnew[name] = data, br, headers
+    if not wwwnew:
+        raise ServerError(
+            "Web frontend missing. Did you forget npm run build?",
+            extra={"wwwroot": str(base)},
+            quiet=True,
+        )
     return wwwnew
 
 
-@app.add_task
+@app.before_server_start
+async def start(app):
+    await load_wwwroot(app)
+    if app.debug:
+        app.add_task(refresh_wwwroot())
+
+
+async def load_wwwroot(app):
+    global www
+    www = await asyncio.get_event_loop().run_in_executor(
+        app.ctx.threadexec, _load_wwwroot, www
+    )
+
+
 async def refresh_wwwroot():
     while True:
+        await asyncio.sleep(0.5)
         try:
             wwwold = www
-            await load_wwwroot()
+            await load_wwwroot(app)
             changes = ""
             for name in sorted(www):
                 attr = www[name]
@@ -151,7 +163,6 @@ async def refresh_wwwroot():
             print("Error loading wwwroot", e)
         if not app.debug:
             return
-        await asyncio.sleep(0.5)
 
 
 @app.route("/<path:path>", methods=["GET", "HEAD"])
@@ -166,66 +177,70 @@ async def wwwroot(req, path=""):
         return empty(304, headers=headers)
     # Brotli compressed?
     if br and "br" in req.headers.accept_encoding.split(", "):
-        headers = {
-            **headers,
-            "content-encoding": "br",
-        }
+        headers = {**headers, "content-encoding": "br"}
         data = br
     return raw(data, headers=headers)
 
 
+def get_files(wanted: set) -> list[tuple[PurePosixPath, Path]]:
+    loc = PurePosixPath()
+    idx = 0
+    ret = []
+    level: int | None = None
+    parent: PurePosixPath | None = None
+    with watching.state.lock:
+        root = watching.state.root
+        while idx < len(root):
+            f = root[idx]
+            loc = PurePosixPath(*loc.parts[: f.level - 1]) / f.name
+            if parent is not None and f.level <= level:
+                level = parent = None
+            if f.key in wanted:
+                level, parent = f.level, loc.parent
+            if parent is not None:
+                wanted.discard(f.key)
+                ret.append((loc.relative_to(parent), watching.rootpath / loc))
+            idx += 1
+    return ret
+
+
 @app.get("/zip/<keys>/<zipfile:ext=zip>")
 async def zip_download(req, keys, zipfile, ext):
     """Download a zip archive of the given keys"""
+
     wanted = set(keys.split("+"))
-    with watching.tree_lock:
-        q = deque([([], None, watching.tree[""].dir)])
-        files = []
-        while q:
-            locpar, relpar, d = q.pop()
-            for name, attr in d.items():
-                loc = [*locpar, name]
-                rel = None
-                if relpar or attr.key in wanted:
-                    rel = [*relpar, name] if relpar else [name]
-                    wanted.discard(attr.key)
-                isdir = isinstance(attr, DirEntry)
-                if isdir:
-                    q.append((loc, rel, attr.dir))
-                if rel:
-                    files.append(
-                        ("/".join(rel), Path(watching.rootpath.joinpath(*loc)))
-                    )
+    files = get_files(wanted)
 
     if not files:
         raise NotFound(
             "No files found",
-            context={"keys": keys, "zipfile": zipfile, "wanted": wanted},
+            context={"keys": keys, "zipfile": f"{zipfile}.{ext}", "wanted": wanted},
         )
     if wanted:
         raise NotFound("Files not found", context={"missing": wanted})
 
-    files = natsorted(files, key=lambda f: f[0], alg=ns.IGNORECASE)
-
     def local_files(files):
         for rel, p in files:
             s = p.stat()
             size = s.st_size
             modified = datetime.datetime.fromtimestamp(s.st_mtime, datetime.UTC)
+            name = rel.as_posix()
             if p.is_dir():
-                yield rel, modified, S_IFDIR | 0o755, ZIP_AUTO(size), b""
+                yield f"{name}/", modified, S_IFDIR | 0o755, ZIP_AUTO(size), iter(b"")
             else:
-                yield rel, modified, S_IFREG | 0o644, ZIP_AUTO(size), contents(p)
+                yield name, modified, S_IFREG | 0o644, ZIP_AUTO(size), contents(p, size)
 
-    def contents(name):
+    def contents(name, size):
         with name.open("rb") as f:
-            while chunk := f.read(65536):
+            while size > 0 and (chunk := f.read(min(size, 1 << 20))):
+                size -= len(chunk)
                 yield chunk
+        assert size == 0
 
     def worker():
         try:
             for chunk in stream_zip(local_files(files)):
-                asyncio.run_coroutine_threadsafe(queue.put(chunk), loop)
+                asyncio.run_coroutine_threadsafe(queue.put(chunk), loop).result()
         except Exception:
             logging.exception("Error streaming ZIP")
             raise
@@ -238,7 +253,10 @@ async def zip_download(req, keys, zipfile, ext):
     thread = loop.run_in_executor(app.ctx.threadexec, worker)
 
     # Stream the response
-    res = await req.respond(content_type="application/zip")
+    res = await req.respond(
+        content_type="application/zip",
+        headers={"cache-control": "no-store"},
+    )
     while chunk := await queue.get():
         await res.send(chunk)
 

cista/auth.py

@@ -68,10 +68,10 @@ def verify(request, *, privileged=False):
         if request.ctx.user:
             if request.ctx.user.privileged:
                 return
-            raise Forbidden("Access Forbidden: Only for privileged users")
+            raise Forbidden("Access Forbidden: Only for privileged users", quiet=True)
     elif config.config.public or request.ctx.user:
         return
-    raise Unauthorized("Login required", "cookie")
+    raise Unauthorized("Login required", "cookie", quiet=True)
 
 
 bp = Blueprint("auth")

cista/protocol.py

@@ -112,47 +112,36 @@ class ErrorMsg(msgspec.Struct):
 ## Directory listings
 
 
-class FileEntry(msgspec.Struct):
-    key: str
-    size: int
-    mtime: int
-
-
-class DirEntry(msgspec.Struct):
-    key: str
-    size: int
-    mtime: int
-    dir: DirList
-
-    def __getitem__(self, name):
-        return self.dir[name]
-
-    def __setitem__(self, name, value):
-        self.dir[name] = value
-
-    def __contains__(self, name):
-        return name in self.dir
-
-    def __delitem__(self, name):
-        del self.dir[name]
-
-    @property
-    def props(self):
-        return {k: v for k, v in self.__struct_fields__ if k != "dir"}
-
-
-DirList = dict[str, FileEntry | DirEntry]
-
-
-class UpdateEntry(msgspec.Struct, omit_defaults=True):
-    """Updates the named entry in the tree. Fields that are set replace old values. A list of entries recurses directories."""
-
+class FileEntry(msgspec.Struct, array_like=True):
+    level: int
     name: str
     key: str
-    deleted: bool = False
-    size: int | None = None
-    mtime: int | None = None
-    dir: DirList | None = None
+    mtime: int
+    size: int
+    isfile: int
+
+
+class Update(msgspec.Struct, array_like=True):
+    ...
+
+
+class UpdKeep(Update, tag="k"):
+    count: int
+
+
+class UpdDel(Update, tag="d"):
+    count: int
+
+
+class UpdIns(Update, tag="i"):
+    items: list[FileEntry]
+
+
+class Space(msgspec.Struct):
+    disk: int
+    free: int
+    usage: int
+    storage: int
 
 
 def make_dir_data(root):

cista/watching.py

@@ -1,20 +1,132 @@
 import asyncio
 import shutil
+import stat
 import sys
 import threading
 import time
+from os import stat_result
 from pathlib import Path, PurePosixPath
 
 import msgspec
+from natsort import humansorted, natsort_keygen, ns
 from sanic.log import logging
 
 from cista import config
 from cista.fileio import fuid
-from cista.protocol import DirEntry, FileEntry, UpdateEntry
+from cista.protocol import FileEntry, Space, UpdDel, UpdIns, UpdKeep
 
 pubsub = {}
-tree = {"": None}
-tree_lock = threading.Lock()
+sortkey = natsort_keygen(alg=ns.LOCALE)
+
+
+class State:
+    def __init__(self):
+        self.lock = threading.RLock()
+        self._space = Space(0, 0, 0, 0)
+        self._listing: list[FileEntry] = []
+
+    @property
+    def space(self):
+        with self.lock:
+            return self._space
+
+    @space.setter
+    def space(self, space):
+        with self.lock:
+            self._space = space
+
+    @property
+    def root(self) -> list[FileEntry]:
+        with self.lock:
+            return self._listing[:]
+
+    @root.setter
+    def root(self, listing: list[FileEntry]):
+        with self.lock:
+            self._listing = listing
+
+    def _slice(self, idx: PurePosixPath | tuple[PurePosixPath, int]):
+        relpath, relfile = idx if isinstance(idx, tuple) else (idx, 0)
+        begin, end = 0, len(self._listing)
+        level = 0
+        isfile = 0
+        while level < len(relpath.parts):
+            # Enter a subdirectory
+            level += 1
+            begin += 1
+            if level == len(relpath.parts):
+                isfile = relfile
+            name = relpath.parts[level - 1]
+            namesort = sortkey(name)
+            r = self._listing[begin]
+            assert r.level == level
+            # Iterate over items at this level
+            while (
+                begin < end
+                and r.name != name
+                and r.isfile <= isfile
+                and sortkey(r.name) < namesort
+            ):
+                # Skip contents
+                begin += 1
+                while begin < end and self._listing[begin].level > level:
+                    begin += 1
+                # Not found?
+                if begin == end or self._listing[begin].level < level:
+                    return slice(begin, begin)
+                r = self._listing[begin]
+            # Not found?
+            if begin == end or r.name != name:
+                return slice(begin, begin)
+        # Found an item, now find its end
+        for end in range(begin + 1, len(self._listing)):
+            if self._listing[end].level <= level:
+                break
+        return slice(begin, end)
+
+    def __getitem__(self, index: PurePosixPath | tuple[PurePosixPath, int]):
+        with self.lock:
+            return self._listing[self._slice(index)]
+
+    def __setitem__(
+        self, index: tuple[PurePosixPath, int], value: list[FileEntry]
+    ) -> None:
+        rel, isfile = index
+        with self.lock:
+            if rel.parts:
+                parent = self._slice(rel.parent)
+                if parent.start == parent.stop:
+                    raise ValueError(
+                        f"Parent folder {rel.as_posix()} is missing for {rel.name}"
+                    )
+            self._listing[self._slice(index)] = value
+
+    def __delitem__(self, relpath: PurePosixPath):
+        with self.lock:
+            del self._listing[self._slice(relpath)]
+
+    def _index(self, rel: PurePosixPath):
+        idx = 0
+        ret = []
+
+    def _dir(self, idx: int):
+        level = self._listing[idx].level + 1
+        end = len(self._listing)
+        idx += 1
+        ret = []
+        while idx < end and (r := self._listing[idx]).level >= level:
+            if r.level == level:
+                ret.append(idx)
+        return ret, idx
+
+    def update(self, rel: PurePosixPath, value: FileEntry):
+        begin = 0
+        parents = []
+        while self._listing[begin].level < len(rel.parts):
+            parents.append(begin)
+
+
+state = State()
 rootpath: Path = None  # type: ignore
 quit = False
 modified_flags = (
@@ -26,23 +138,21 @@ modified_flags = (
     "IN_MOVED_FROM",
     "IN_MOVED_TO",
 )
-disk_usage = None
 
 
 def watcher_thread(loop):
-    global disk_usage, rootpath
+    global rootpath
     import inotify.adapters
 
     while True:
         rootpath = config.config.path
         i = inotify.adapters.InotifyTree(rootpath.as_posix())
-        old = format_tree() if tree[""] else None
-        with tree_lock:
-            # Initialize the tree from filesystem
-            tree[""] = walk(rootpath)
-        msg = format_tree()
-        if msg != old:
-            asyncio.run_coroutine_threadsafe(broadcast(msg), loop)
+        # Initialize the tree from filesystem
+        old, new = state.root, walk()
+        if old != new:
+            with state.lock:
+                state.root = new
+                broadcast(format_root(new), loop)
 
         # The watching is not entirely reliable, so do a full refresh every minute
         refreshdl = time.monotonic() + 60.0
@@ -52,9 +162,10 @@ def watcher_thread(loop):
                 return
             # Disk usage update
             du = shutil.disk_usage(rootpath)
-            if du != disk_usage:
-                disk_usage = du
-                asyncio.run_coroutine_threadsafe(broadcast(format_du()), loop)
+            space = Space(*du, storage=state.root[0].size)
+            if space != state.space:
+                state.space = space
+                broadcast(format_space(space), loop)
                 break
             # Do a full refresh?
             if time.monotonic() > refreshdl:
@@ -75,144 +186,143 @@ def watcher_thread(loop):
 
 
 def watcher_thread_poll(loop):
-    global disk_usage, rootpath
+    global rootpath
 
     while not quit:
         rootpath = config.config.path
-        old = format_tree() if tree[""] else None
-        with tree_lock:
-            # Initialize the tree from filesystem
-            tree[""] = walk(rootpath)
-        msg = format_tree()
-        if msg != old:
-            asyncio.run_coroutine_threadsafe(broadcast(msg), loop)
+        old = state.root
+        new = walk()
+        if old != new:
+            with state.lock:
+                state.root = new
+                broadcast(format_update(old, new), loop)
 
         # Disk usage update
         du = shutil.disk_usage(rootpath)
-        if du != disk_usage:
-            disk_usage = du
-            asyncio.run_coroutine_threadsafe(broadcast(format_du()), loop)
+        space = Space(*du, storage=state.root[0].size)
+        if space != state.space:
+            state.space = space
+            broadcast(format_space(space), loop)
 
-        time.sleep(1.0)
+        time.sleep(2.0)
 
 
-def format_du():
-    return msgspec.json.encode(
-        {
-            "space": {
-                "disk": disk_usage.total,
-                "used": disk_usage.used,
-                "free": disk_usage.free,
-                "storage": tree[""].size,
-            },
-        },
-    ).decode()
-
-
-def format_tree():
-    root = tree[""]
-    return msgspec.json.encode({"root": root}).decode()
-
-
-def walk(path: Path) -> DirEntry | FileEntry | None:
+def walk(rel=PurePosixPath()) -> list[FileEntry]:  # noqa: B008
+    path = rootpath / rel
     try:
-        s = path.stat()
-        key = fuid(s)
-        assert key, repr(key)
-        mtime = int(s.st_mtime)
-        if path.is_file():
-            return FileEntry(key, s.st_size, mtime)
+        st = path.stat()
+    except OSError:
+        return []
+    return _walk(rel, int(not stat.S_ISDIR(st.st_mode)), st)
 
-        tree = {
-            p.name: v
-            for p in path.iterdir()
-            if not p.name.startswith(".")
-            if (v := walk(p)) is not None
-        }
-        if tree:
-            size = sum(v.size for v in tree.values())
-            mtime = max(mtime, *(v.mtime for v in tree.values()))
-        else:
-            size = 0
-        return DirEntry(key, size, mtime, tree)
+
+def _walk(rel: PurePosixPath, isfile: int, st: stat_result) -> list[FileEntry]:
+    entry = FileEntry(
+        level=len(rel.parts),
+        name=rel.name,
+        key=fuid(st),
+        mtime=int(st.st_mtime),
+        size=st.st_size if isfile else 0,
+        isfile=isfile,
+    )
+    if isfile:
+        return [entry]
+    ret = [entry]
+    path = rootpath / rel
+    try:
+        li = []
+        for f in path.iterdir():
+            if f.name.startswith("."):
+                continue  # No dotfiles
+            s = f.stat()
+            li.append((int(not stat.S_ISDIR(s.st_mode)), f.name, s))
+        for [isfile, name, s] in humansorted(li):
+            subtree = _walk(rel / name, isfile, s)
+            child = subtree[0]
+            entry.mtime = max(entry.mtime, child.mtime)
+            entry.size += child.size
+            ret.extend(subtree)
     except FileNotFoundError:
-        return None
+        pass  # Things may be rapidly in motion
     except OSError as e:
         print("OS error walking path", path, e)
-        return None
+    return ret
 
 
 def update(relpath: PurePosixPath, loop):
     """Called by inotify updates, check the filesystem and broadcast any changes."""
     if rootpath is None or relpath is None:
         print("ERROR", rootpath, relpath)
-    new = walk(rootpath / relpath)
-    with tree_lock:
-        update = update_internal(relpath, new)
-        if not update:
-            return  # No changes
-        msg = msgspec.json.encode({"update": update}).decode()
-        asyncio.run_coroutine_threadsafe(broadcast(msg), loop)
+    new = walk(relpath)
+    with state.lock:
+        old = state[relpath]
+        if old == new:
+            return
+        old = state.root
+        if new:
+            state[relpath, new[0].isfile] = new
+        else:
+            del state[relpath]
+        broadcast(format_update(old, state.root), loop)
 
 
-def update_internal(
-    relpath: PurePosixPath,
-    new: DirEntry | FileEntry | None,
-) -> list[UpdateEntry]:
-    path = "", *relpath.parts
-    old = tree
-    elems = []
-    for name in path:
-        if name not in old:
-            # File or folder created
-            old = None
-            elems.append((name, None))
-            if len(elems) < len(path):
-                # We got a notify for an item whose parent is not in tree
-                print("Tree out of sync DEBUG", relpath)
-                print(elems)
-                print("Current tree:")
-                print(tree[""])
-                print("Walking all:")
-                print(walk(rootpath))
-                raise ValueError("Tree out of sync")
-            break
-        old = old[name]
-        elems.append((name, old))
-    if old == new:
-        return []
-    mt = new.mtime if new else 0
-    szdiff = (new.size if new else 0) - (old.size if old else 0)
-    # Update parents
+def format_update(old, new):
+    # Make keep/del/insert diff until one of the lists ends
+    oidx, nidx = 0, 0
     update = []
-    for name, entry in elems[:-1]:
-        u = UpdateEntry(name, entry.key)
-        if szdiff:
-            entry.size += szdiff
-            u.size = entry.size
-        if mt > entry.mtime:
-            u.mtime = entry.mtime = mt
-        update.append(u)
-    # The last element is the one that changed
-    name, entry = elems[-1]
-    parent = elems[-2][1] if len(elems) > 1 else tree
-    u = UpdateEntry(name, new.key if new else entry.key)
-    if new:
-        parent[name] = new
-        if u.size != new.size:
-            u.size = new.size
-        if u.mtime != new.mtime:
-            u.mtime = new.mtime
-        if isinstance(new, DirEntry) and u.dir != new.dir:
-            u.dir = new.dir
-    else:
-        del parent[name]
-        u.deleted = True
-    update.append(u)
-    return update
+    keep_count = 0
+    while oidx < len(old) and nidx < len(new):
+        if old[oidx] == new[nidx]:
+            keep_count += 1
+            oidx += 1
+            nidx += 1
+            continue
+        if keep_count > 0:
+            update.append(UpdKeep(keep_count))
+            keep_count = 0
+
+        del_count = 0
+        rest = new[nidx:]
+        while old[oidx] not in rest:
+            del_count += 1
+            oidx += 1
+
+        if del_count:
+            update.append(UpdDel(del_count))
+            oidx += 1
+            continue
+
+        insert_items = []
+        rest = old[oidx:]
+        while nidx < len(new) and new[nidx] not in rest:
+            insert_items.append(new[nidx])
+            nidx += 1
+        update.append(UpdIns(insert_items))
+
+    # Diff any remaining
+    if keep_count > 0:
+        update.append(UpdKeep(keep_count))
+    if oidx < len(old):
+        update.append(UpdDel(len(old) - oidx))
+    elif nidx < len(new):
+        update.append(UpdIns(new[nidx:]))
+
+    return msgspec.json.encode({"update": update}).decode()
 
 
-async def broadcast(msg):
+def format_space(usage):
+    return msgspec.json.encode({"space": usage}).decode()
+
+
+def format_root(root):
+    return msgspec.json.encode({"root": root}).decode()
+
+
+def broadcast(msg, loop):
+    return asyncio.run_coroutine_threadsafe(abroadcast(msg), loop).result()
+
+
+async def abroadcast(msg):
     try:
         for queue in pubsub.values():
             queue.put_nowait(msg)

frontend/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "front",
+  "name": "cista-frontend",
   "version": "0.0.0",
   "private": true,
   "scripts": {

frontend/src/App.vue

@@ -1,13 +1,13 @@
 <template>
   <LoginModal />
   <header>
-    <HeaderMain ref="headerMain" :path="path.pathList">
+    <HeaderMain ref="headerMain" :path="path.pathList" :query="path.query">
       <HeaderSelected :path="path.pathList" />
     </HeaderMain>
     <BreadCrumb :path="path.pathList" tabindex="-1"/>
   </header>
   <main>
-    <RouterView :path="path.pathList" />
+    <RouterView :path="path.pathList" :query="path.query" />
   </main>
 </template>
 
@@ -16,7 +16,7 @@ import { RouterView } from 'vue-router'
 import type { ComputedRef } from 'vue'
 import type HeaderMain from '@/components/HeaderMain.vue'
 import { onMounted, onUnmounted, ref, watchEffect } from 'vue'
-import { watchConnect, watchDisconnect } from '@/repositories/WS'
+import { loadSession, watchConnect, watchDisconnect } from '@/repositories/WS'
 import { useDocumentStore } from '@/stores/documents'
 
 import { computed } from 'vue'
@@ -25,19 +25,23 @@ import Router from '@/router/index'
 interface Path {
   path: string
   pathList: string[]
+  query: string
 }
 const documentStore = useDocumentStore()
 const path: ComputedRef<Path> = computed(() => {
-  const p = decodeURIComponent(Router.currentRoute.value.path)
-  const pathList = p.split('/').filter(value => value !== '')
+  const p = decodeURIComponent(Router.currentRoute.value.path).split('//')
+  const pathList = p[0].split('/').filter(value => value !== '')
+  const query = p.slice(1).join('//')
   return {
-    path: p,
-    pathList
+    path: p[0],
+    pathList,
+    query
   }
 })
 watchEffect(() => {
   document.title = path.value.path.replace(/\/$/, '').split('/').pop() || documentStore.server.name || 'Cista Storage'
 })
+onMounted(loadSession)
 onMounted(watchConnect)
 onUnmounted(watchDisconnect)
 // Update human-readable x seconds ago messages from mtimes

frontend/src/components/BreadCrumb.vue

@@ -46,8 +46,11 @@ const isCurrent = (index: number) => index == props.path.length ? 'location' : u
 const navigate = (index: number) => {
   const link = links[index]
   if (!link) throw Error(`No link at index ${index} (path: ${props.path})`)
+  const url = `/${longest.value.slice(0, index).join('/')}/`
+  const here = `/${longest.value.join('/')}/`
   link.focus()
-  router.replace(`/${longest.value.slice(0, index).join('/')}`)
+  if (here.startsWith(location.hash.slice(1))) router.replace(url)
+  else router.push(url)
 }
 
 const move = (dir: number) => {

frontend/src/components/FileExplorer.vue

@@ -3,34 +3,11 @@
     <thead>
       <tr>
         <th class="selection">
-          <input
-            type="checkbox"
-            tabindex="-1"
-            v-model="allSelected"
-            :indeterminate="selectionIndeterminate"
-          />
-        </th>
-        <th
-          class="sortcolumn"
-          :class="{ sortactive: sort === 'name' }"
-          @click="toggleSort('name')"
-        >
-          Name
-        </th>
-        <th
-          class="sortcolumn modified right"
-          :class="{ sortactive: sort === 'modified' }"
-          @click="toggleSort('modified')"
-        >
-          Modified
-        </th>
-        <th
-          class="sortcolumn size right"
-          :class="{ sortactive: sort === 'size' }"
-          @click="toggleSort('size')"
-        >
-          Size
+          <input type="checkbox" tabindex="-1" v-model="allSelected" :indeterminate="selectionIndeterminate">
         </th>
+        <th class="sortcolumn" :class="{ sortactive: sort === 'name' }" @click="toggleSort('name')">Name</th>
+        <th class="sortcolumn modified right" :class="{ sortactive: sort === 'modified' }" @click="toggleSort('modified')">Modified</th>
+        <th class="sortcolumn size right" :class="{ sortactive: sort === 'size' }" @click="toggleSort('size')">Size</th>
         <th class="menu"></th>
       </tr>
     </thead>
@@ -38,27 +15,13 @@
       <tr v-if="editing?.key === 'new'" class="folder">
         <td class="selection"></td>
         <td class="name">
-          <FileRenameInput
-            :doc="editing"
-            :rename="mkdir"
-            :exit="
-              () => {
-                editing = null
-              }
-            "
-          />
+          <FileRenameInput :doc="editing" :rename="mkdir" :exit="() => {editing = null}" />
         </td>
-        <td class="modified right">
-          <time :datetime="new Date(editing.mtime).toISOString().replace('.000', '')">{{
-            editing.modified
-          }}</time>
-        </td>
-        <td class="size right">{{ editing.sizedisp }}</td>
+        <FileModified :doc=editing />
+        <FileSize :doc=editing />
         <td class="menu"></td>
       </tr>
-      <template
-        v-for="(doc, index) in sortedDocuments"
-        :key="doc.key">
+      <template v-for="(doc, index) in sortedDocuments" :key="doc.key">
         <tr class="folder-change" v-if="showFolderBreadcrumb(index)">
           <th colspan="5"><BreadCrumb :path="doc.loc ? doc.loc.split('/') : []" /></th>
         </tr>
@@ -82,16 +45,9 @@
             />
           </td>
           <td class="name">
-            <template v-if="editing === doc"
-              ><FileRenameInput
-                :doc="doc"
-                :rename="rename"
-                :exit="
-                  () => {
-                    editing = null
-                  }
-                "
-            /></template>
+            <template v-if="editing === doc">
+              <FileRenameInput :doc="doc" :rename="rename" :exit="() => {editing = null}" />
+            </template>
             <template v-else>
               <a
                 :href="url_for(doc)"
@@ -102,29 +58,13 @@
                 @keyup.right.stop="ev => { if (doc.dir) (ev.target as HTMLElement).click() }"
                 >{{ doc.name }}</a
               >
-              <button
-                v-if="cursor == doc"
-                class="rename-button"
-                @click="() => (editing = doc)"
-              >
-                🖊️
-              </button>
+              <button v-if="cursor == doc" class="rename-button" @click="() => (editing = doc)">🖊️</button>
             </template>
           </td>
-          <td class="modified right">
-            <time
-              :data-tooltip="new Date(1000 * doc.mtime).toISOString().replace('T', '\n').replace('.000Z', ' UTC')"
-              >{{ doc.modified }}</time
-            >
-          </td>
-          <td class="size right">{{ doc.sizedisp }}</td>
+          <FileModified :doc=doc />
+          <FileSize :doc=doc />
           <td class="menu">
-            <button
-              tabindex="-1"
-              @click.stop="contextMenu($event, doc)"
-            >
-              ⋮
-            </button>
+            <button tabindex="-1" @click.stop="contextMenu($event, doc)">⋮</button>
           </td>
         </tr>
       </template>
@@ -147,13 +87,10 @@ import { connect, controlUrl } from '@/repositories/WS'
 import { collator, formatSize, formatUnixDate } from '@/utils'
 import { useRouter } from 'vue-router'
 
-const props = withDefaults(
-  defineProps<{
-    path: Array<string>
-    documents: Document[]
-  }>(),
-  {}
-)
+const props = defineProps<{
+  path: Array<string>
+  documents: Document[]
+}>()
 const documentStore = useDocumentStore()
 const router = useRouter()
 const url_for = (doc: Document) => {
@@ -309,7 +246,7 @@ const mkdir = (doc: Document, name: string) => {
         editing.value = null
       } else {
         console.log('mkdir', msg)
-        router.push(`/${doc.loc}/${name}/`)
+        router.push(doc.loc ? `/${doc.loc}/${name}/` : `/${name}/`)
       }
     }
   })
@@ -400,7 +337,7 @@ table .selection {
   text-overflow: clip;
 }
 table .modified {
-  width: 8em;
+  width: 9em;
 }
 table .size {
   width: 5em;

frontend/src/components/FileModified.vue

@@ -0,0 +1,22 @@
+<template>
+  <td class="modified right">
+    <time :data-tooltip=tooltip :datetime=datetime>{{ doc.modified }}</time>
+  </td>
+</template>
+
+<script setup lang="ts">
+import type { Document } from '@/repositories/Document'
+import { computed } from 'vue'
+
+const datetime = computed(() =>
+  new Date(1000 * props.doc.mtime).toISOString().replace('.000Z', 'Z')
+)
+
+const tooltip = computed(() =>
+  datetime.value.replace('T', '\n').replace('Z', ' UTC')
+)
+
+const props = defineProps<{
+    doc: Document
+}>()
+</script>

frontend/src/components/FileSize.vue

@@ -0,0 +1,43 @@
+<template>
+  <td class="size right" :class=sizeClass>{{ doc.sizedisp }}</td>
+</template>
+
+<script setup lang="ts">
+import type { Document } from '@/repositories/Document'
+import { computed } from 'vue'
+
+const sizeClass = computed(() => {
+  const unit = props.doc.sizedisp.split('\u202F').slice(-1)[0]
+  return +unit ? "bytes" : unit
+})
+
+const props = defineProps<{
+    doc: Document
+}>()
+</script>
+
+<style scoped>
+.size.empty { color: #555 }
+.size.bytes { color: #77a }
+.size.kB { color: #474 }
+.size.MB { color: #a80 }
+.size.GB { color: #f83 }
+.size.TB, .size.PB, .size.EB, .size.huge {
+  color: #f44;
+  text-shadow: 0 0 .2em;
+}
+
+@media (prefers-color-scheme: dark) {
+  .size.empty { color: #bbb }
+  .size.bytes { color: #99d }
+  .size.kB { color: #aea }
+  .size.MB { color: #ff4 }
+  .size.GB { color: #f86 }
+  .size.TB, .size.PB, .size.EB, .size.huge { color: #f55 }
+}
+
+.cursor .size {
+  color: inherit;
+  text-shadow: none;
+}
+</style>

frontend/src/components/HeaderMain.vue

@@ -17,7 +17,9 @@
         <input
           ref="search"
           type="search"
-          v-model="documentStore.search"
+          :value="query"
+          @blur="ev => { if (!query) closeSearch(ev) }"
+          @input="updateSearch"
           placeholder="Search words"
           class="margin-input"
           @keyup.escape="closeSearch"
@@ -31,20 +33,30 @@
 
 <script setup lang="ts">
 import { useDocumentStore } from '@/stores/documents'
-import { ref, nextTick } from 'vue'
+import { ref, nextTick, watchEffect } from 'vue'
 import ContextMenu from '@imengyu/vue3-context-menu'
+import router from '@/router';
 
 const documentStore = useDocumentStore()
 const showSearchInput = ref<boolean>(false)
 const search = ref<HTMLInputElement | null>()
 const searchButton = ref<HTMLButtonElement | null>()
 
-const closeSearch = () => {
+const closeSearch = ev => {
   if (!showSearchInput.value) return  // Already closing
   showSearchInput.value = false
-  documentStore.search = ''
   const breadcrumb = document.querySelector('.breadcrumb') as HTMLElement
   breadcrumb.focus()
+  updateSearch(ev)
+}
+const updateSearch = ev => {
+  const q = ev.target.value
+  let p = props.path.join('/')
+  p = p ? `/${p}` : ''
+  const url = q ? `${p}//${q}` : (p || '/')
+  console.log("Update search", url)
+  if (!props.query && q) router.push(url)
+  else router.replace(url)
 }
 const toggleSearchInput = () => {
   showSearchInput.value = !showSearchInput.value
@@ -54,7 +66,9 @@ const toggleSearchInput = () => {
     if (input) input.focus()
   })
 }
-
+watchEffect(() => {
+  if (props.query) showSearchInput.value = true
+})
 const settingsMenu = (e: Event) => {
   // show the context menu
   const items = []
@@ -69,9 +83,10 @@ const settingsMenu = (e: Event) => {
     items,
   })
 }
-const props = defineProps({
+const props = defineProps<{
   path: Array<string>
-})
+  query: string
+}>()
 
 defineExpose({
   toggleSearchInput,

frontend/src/repositories/Document.ts

@@ -22,29 +22,16 @@ export type errorEvent = {
 
 // Raw types the backend /api/watch sends us
 
-export type FileEntry = {
-  key: FUID
-  size: number
-  mtime: number
-}
+export type FileEntry = [
+  number,  // level
+  string,  // name
+  FUID,
+  number, //mtime
+  number, // size
+  number, // isfile
+]
 
-export type DirEntry = {
-  key: FUID
-  size: number
-  mtime: number
-  dir: DirList
-}
-
-export type DirList = Record<string, FileEntry | DirEntry>
-
-export type UpdateEntry = {
-  name: string
-  deleted?: boolean
-  key?: FUID
-  size?: number
-  mtime?: number
-  dir?: DirList
-}
+export type UpdateEntry = ['k', number] | ['d', number] | ['i', Array<FileEntry>]
 
 // Helper structure for selections
 export interface SelectedItems {

frontend/src/repositories/WS.ts

@@ -1,14 +1,29 @@
 import { useDocumentStore } from "@/stores/documents"
-import type { DirEntry, UpdateEntry, errorEvent } from "./Document"
+import type { FileEntry, UpdateEntry, errorEvent } from "./Document"
 
 export const controlUrl = '/api/control'
 export const uploadUrl = '/api/upload'
 export const watchUrl = '/api/watch'
 
-let tree = null as DirEntry | null
+let tree = [] as FileEntry[]
 let reconnectDuration = 500
 let wsWatch = null as WebSocket | null
 
+export const loadSession = () => {
+  const store = useDocumentStore()
+  try {
+    tree = JSON.parse(sessionStorage["cista-files"])
+    store.updateRoot(tree)
+    return true
+  } catch (error) {
+    return false
+  }
+}
+
+const saveSession = () => {
+  sessionStorage["cista-files"] = JSON.stringify(tree)
+}
+
 export const connect = (path: string, handlers: Partial<Record<keyof WebSocketEventMap, any>>) => {
   const webSocket = new WebSocket(new URL(path, location.origin.replace(/^http/, 'ws')))
   for (const [event, handler] of Object.entries(handlers)) webSocket.addEventListener(event, handler)
@@ -99,29 +114,31 @@ function handleRootMessage({ root }: { root: DirEntry }) {
   console.log('Watch root', root)
   store.updateRoot(root)
   tree = root
+  saveSession()
 }
 
 function handleUpdateMessage(updateData: { update: UpdateEntry[] }) {
   const store = useDocumentStore()
-  console.log('Watch update', updateData.update)
+  const update = updateData.update
+  console.log('Watch update', update)
   if (!tree) return console.error('Watch update before root')
-  let node: DirEntry = tree
-  for (const elem of updateData.update) {
-    if (elem.deleted) {
-      delete node.dir[elem.name]
-      break // Deleted elements can't have further children
-    }
-    if (elem.name) {
-      // @ts-ignore
-      console.log(node, elem.name)
-      node = node.dir[elem.name] ||= {}
-    }
-    if (elem.key !== undefined) node.key = elem.key
-    if (elem.size !== undefined) node.size = elem.size
-    if (elem.mtime !== undefined) node.mtime = elem.mtime
-    if (elem.dir !== undefined) node.dir = elem.dir
+  let newtree = []
+  let oidx = 0
+
+  for (const [action, arg] of update) {
+      if (action === 'k') {
+          newtree.push(...tree.slice(oidx, oidx + arg))
+          oidx += arg
+      }
+      else if (action === 'd') oidx += arg
+      else if (action === 'i') newtree.push(...arg)
+      else console.log("Unknown update action", action, arg)
   }
-  store.updateRoot(tree)
+  if (oidx != tree.length)
+    throw Error(`Tree update out of sync, number of entries mismatch: got ${oidx}, expected ${tree.length}`)
+  store.updateRoot(newtree)
+  tree = newtree
+  saveSession()
 }
 
 function handleError(msg: errorEvent) {

frontend/src/stores/documents.ts

@@ -1,15 +1,10 @@
-import type {
-  Document,
-  DirEntry,
-  FileEntry,
-  FUID,
-  SelectedItems
-} from '@/repositories/Document'
+import type { Document, FileEntry, FUID, SelectedItems } from '@/repositories/Document'
 import { formatSize, formatUnixDate, haystackFormat } from '@/utils'
 import { defineStore } from 'pinia'
 import { collator } from '@/utils'
 import { logoutUser } from '@/repositories/User'
 import { watchConnect } from '@/repositories/WS'
+import { format } from 'path'
 
 type FileData = { id: string; mtime: number; size: number; dir: DirectoryData }
 type DirectoryData = {
@@ -26,7 +21,6 @@ export const useDocumentStore = defineStore({
   id: 'documents',
   state: () => ({
     document: [] as Document[],
-    search: "" as string,
     selected: new Set<FUID>(),
     uploadingDocuments: [],
     uploadCount: 0 as number,
@@ -41,48 +35,31 @@ export const useDocumentStore = defineStore({
       isOpenLoginModal: false
     } as User
   }),
-  persist: {
-    storage: sessionStorage,
-    paths: ['document'],
-  },
   actions: {
-    updateRoot(root: DirEntry | null = null) {
-      if (!root) {
-        this.document = []
-        return
-      }
-      // Transform tree data to flat documents array
-      let loc = ""
-      const mapper = ([name, attr]: [string, FileEntry | DirEntry]) => ({
-        ...attr,
-        loc,
-        name,
-        sizedisp: formatSize(attr.size),
-        modified: formatUnixDate(attr.mtime),
-        haystack: haystackFormat(name),
-      })
-      const queue = [...Object.entries(root.dir ?? {}).map(mapper)]
+    updateRoot(root: FileEntry[]) {
       const docs = []
-      for (let doc; (doc = queue.shift()) !== undefined;) {
-        docs.push(doc)
-        if ("dir" in doc) {
-          // Recurse but replace recursive structure with boolean
-          loc = doc.loc ? `${doc.loc}/${doc.name}` : doc.name
-          queue.push(...Object.entries(doc.dir).map(mapper))
-          // @ts-ignore
-          doc.dir = true
-        }
-        // @ts-ignore
-        else doc.dir = false
+      let loc = [] as string[]
+      for (const [level, name, key, mtime, size, isfile] of root) {
+        loc = loc.slice(0, level - 1)
+        docs.push({
+          name,
+          loc: level ? loc.join('/') : '/',
+          key,
+          size,
+          sizedisp: formatSize(size),
+          mtime,
+          modified: formatUnixDate(mtime),
+          haystack: haystackFormat(name),
+          dir: !isfile,
+        })
+        loc.push(name)
       }
-      // Pre sort directory entries folders first then files, names in natural ordering
-      docs.sort((a, b) =>
-        // @ts-ignore
-        b.dir - a.dir ||
-        collator.compare(a.name, b.name)
-      )
+      console.log("Documents", docs)
       this.document = docs as Document[]
     },
+    updateModified() {
+      for (const doc of this.document) doc.modified = formatUnixDate(doc.mtime)
+    },
     login(username: string, privileged: boolean) {
       this.user.username = username
       this.user.privileged = privileged

frontend/src/views/ExplorerView.vue

@@ -16,17 +16,17 @@ import { needleFormat, localeIncludes, collator } from '@/utils';
 
 const documentStore = useDocumentStore()
 const fileExplorer = ref()
-const props = defineProps({
+const props = defineProps<{
   path: Array<string>
-})
+  query: string
+}>()
 const documents = computed(() => {
-  if (!props.path) return []
   const loc = props.path.join('/')
+  const query = props.query
   // List the current location
-  if (!documentStore.search) return documentStore.document.filter(doc => doc.loc === loc)
+  if (!query) return documentStore.document.filter(doc => doc.loc === loc)
   // Find up to 100 newest documents that match the search
-  const search = documentStore.search
-  const needle = needleFormat(search)
+  const needle = needleFormat(query)
   let limit = 100
   let docs = []
   for (const doc of documentStore.recentDocuments) {
@@ -46,7 +46,7 @@ const documents = computed(() => {
     // @ts-ignore
     (a.type === 'file') - (b.type === 'file') ||
     // @ts-ignore
-    b.name.includes(search) - a.name.includes(search) ||
+    b.name.includes(query) - a.name.includes(query) ||
     collator.compare(a.name, b.name)
   ))
   return docs