Refactor with its own entry point and startup script cista, instead of running via sanic. Config file handling and Droppy updates. HTTP redirection/acme server added.

README.md

@@ -2,9 +2,7 @@
 
 Run directly from repository with Hatch (or use pip install as usual):
 ```sh
-hatch run sanic cista --reload --dev
+hatch run cista -l :3000 /path/to/files
 ```
 
-Configuration file is created `.cista.toml` in current directory, which is also shared by default. Edit while the server is not running to set share path and other parameters.
+Settings incl. these arguments are stored to config file on the first startup and later `hatch run cista` is sufficient. If the `cista` script is missing, consider `pip install -e .` (within `hatch shell`) or some other trickery (known issue with installs made prior to adding the startup script).
-
-No authentication yet, so implement access control externally or be careful with your files.

cista/__init__.py

@@ -1 +0,0 @@
-from .app import app

cista/__main__.py

@@ -0,0 +1,92 @@
+import re
+import sys
+from pathlib import Path
+
+from docopt import docopt
+
+from . import app, config, droppy, serve
+from ._version import version
+
+app   # Needed for Sanic multiprocessing
+
+doc = f"""Cista {version} - A file storage for the web.
+
+Usage:
+  cista [-c <confdir>] [-l <host>] [--import-droppy] [--dev] [<path>]
+
+Options:
+  -c CONFDIR        Custom config directory
+  -l LISTEN-ADDR    Listen on
+                       :8000 (localhost port, plain http)
+                       <addr>:3000 (bind another address, port)
+                       /path/to/unix.sock (unix socket)
+                       example.com (run on 80 and 443 with LetsEncrypt)
+  --import-droppy   Import Droppy config from ~/.droppy/config
+  --dev             Developer mode (reloads, friendlier crashes, more logs)
+
+Listen address, path and imported options are preserved in config, and only
+custom config dir and dev mode need to be specified on subsequent runs.
+"""
+
+def main():
+    # Dev mode doesn't catch exceptions
+    if "--dev" in sys.argv:
+        return _main()
+    # Normal mode keeps it quiet
+    try:
+        return _main()
+    except Exception as e:
+        print("Error:", e)
+        return 1
+
+def _main():
+    args = docopt(doc)
+    listen = args["-l"]
+    # Validate arguments first
+    if args["<path>"]:
+        path = Path(args["<path>"])
+        if not path.is_dir():
+            raise ValueError(f"No such directory: {path}")
+    else:
+        path = None
+    if args["-c"]:
+        # Custom config directory
+        confdir = Path(args["-c"]).resolve()
+        if confdir.exists() and not confdir.is_dir():
+            if confdir.name != config.conffile.name:
+                raise ValueError("Config path is not a directory")
+            # Accidentally pointed to the cista.toml, use parent
+            confdir = confdir.parent
+        config.conffile = config.conffile.with_parent(confdir)
+    exists = config.conffile.exists()
+    import_droppy = args["--import-droppy"]
+    necessary_opts = exists or import_droppy or path and listen
+    if not necessary_opts:
+        # Maybe run without arguments
+        print(doc)
+        print("No config file found! Get started with:\n  cista -l :8000 /path/to/files, or\n  cista -l example.com --import-droppy  # Uses Droppy files\n")
+        return 1
+    settings = {}
+    if import_droppy:
+        if exists:
+            raise ValueError(f"Importing Droppy: First remove the existing configuration:\n  rm {config.conffile}")
+        settings = droppy.readconf()
+    if path: settings["path"] = path
+    if listen: settings["listen"] = listen
+    operation = config.update_config(settings)
+    print(f"Config {operation}: {config.conffile}")
+    # Prepare to serve
+    domain = unix = port = None
+    url, _ = serve.parse_listen(config.config.listen)
+    if not config.config.path.is_dir():
+        raise ValueError(f"No such directory: {config.config.path}")
+    extra = f" ({unix})" if unix else ""
+    dev = args["--dev"]
+    if dev:
+        extra += " (dev mode)"
+    print(f"Serving {config.config.path} at {url}{extra}")
+    # Run the server
+    serve.run(dev=dev)
+
+if __name__ == "__main__":
+    sys.exit(main())

cista/app.py

@@ -6,10 +6,9 @@ from sanic import Sanic, redirect
 from sanic.log import logger
 from sanic.response import html
 
-from . import session, watching
+from . import config, session, watching
 from .auth import authbp
-from .config import config
+from .fileio import FileServer
-from .fileio import ROOT, FileServer
 from .protocol import ErrorMsg, FileRange, StatusMsg
 
 app = Sanic("cista")
@@ -22,6 +21,9 @@ def asend(ws, msg):
 
 @app.before_server_start
 async def start_fileserver(app, _):
+    config.load_config()  # Main process may have loaded it but we haven't
+    app.static("/files", config.config.path, use_content_range=True, stream_large_files=True, directory_view=True)
+
     await fileserver.start()
 
 @app.after_server_stop
@@ -30,8 +32,7 @@ async def stop_fileserver(app, _):
 
 @app.get("/")
 async def index_page(request):
-    s = config.public or session.get(request)
+    s = config.config.public or session.get(request)
-    print("Main session", s)
     if not s:
         return redirect("/login")
     index = files("cista").joinpath("static", "index.html").read_text()
@@ -40,8 +41,6 @@ async def index_page(request):
         index += str(E.div(flash, id="flash"))
     return html(index)
 
-app.static("/files", ROOT, use_content_range=True, stream_large_files=True, directory_view=True)
-
 @app.websocket('/api/upload')
 async def upload(request, ws):
     alink = fileserver.alink

cista/auth.py

@@ -8,8 +8,7 @@ import msgspec
 from html5tagger import Document
 from sanic import Blueprint, html, json, redirect
 
-from . import session
+from . import config, session
-from .config import User, config
 
 _argon = argon2.PasswordHasher()
 _droppyhash = re.compile(r'^([a-f0-9]{64})\$([a-f0-9]{8})$')
@@ -21,7 +20,7 @@ def login(username: str, password: str):
     un = _pwnorm(username)
     pw = _pwnorm(password)
     try:
-        u = config.users[un.decode()]
+        u = config.config.users[un.decode()]
     except KeyError:
         raise ValueError("Invalid username")
     # Verify password
@@ -44,12 +43,12 @@ def login(username: str, password: str):
             need_rehash = True
     # Login successful
     if need_rehash:
-        u.set_password(password)
+        set_password(u, password)
     now = int(time())
     u.lastSeen = now
     return u
 
-def set_password(user: User, password: str):
+def set_password(user: config.User, password: str):
     user.hash = _argon.hash(_pwnorm(password))
 
 class LoginResponse(msgspec.Struct):
@@ -90,7 +89,6 @@ async def login_post(request):
         username = request.json["username"]
         password = request.json["password"]
     else:
-        print(request.form)
         username = request.form["username"][0]
         password = request.form["password"][0]
     if not username or not password:

cista/config.py

@@ -10,7 +10,8 @@ import msgspec
 
 
 class Config(msgspec.Struct):
-    path: Path = Path.cwd()
+    path: Path
+    listen: str
     secret: str = secrets.token_hex(12)
     public: bool = False
     users: dict[str, User] = {}
@@ -26,7 +27,8 @@ class Link(msgspec.Struct, omit_defaults=True):
     creator: str = ""
     expires: int = 0
 
-config = Config()
+config = None
+conffile = Path.home() / ".local/share/cista/db.toml"
 
 def derived_secret(*params, len=8) -> bytes:
     """Used to derive secret keys from the main secret"""
@@ -51,10 +53,10 @@ def dec_hook(typ, obj):
         return Path(obj)
     raise TypeError
 
-conffile = Path.cwd() / ".cista.toml"
-
 def config_update(modify):
     global config
+    if not conffile.exists():
+        conffile.parent.mkdir(parents=True, exist_ok=True)
     tmpname = conffile.with_suffix(".tmp")
     try:
         f = tmpname.open("xb")
@@ -68,8 +70,12 @@ def config_update(modify):
             old = conffile.read_bytes()
             c = msgspec.toml.decode(old, type=Config, dec_hook=dec_hook)
         except FileNotFoundError:
+            # No existing config file, make sure we have a folder...
+            confdir = conffile.parent
+            confdir.mkdir(parents=True, exist_ok=True)
+            confdir.chmod(0o700)
             old = b""
-            c = Config()  # Initialize with defaults
+            c = None
         c = modify(c)
         new = msgspec.toml.encode(c, enc_hook=enc_hook)
         if old == new:
@@ -98,12 +104,14 @@ def modifies_config(modify):
         return c
     return wrapper
 
-@modifies_config
+def load_config():
-def droppy_import(config: Config) -> Config:
+    global config
-    p = Path.home() / ".droppy/config"
+    config = msgspec.toml.decode(conffile.read_bytes(), type=Config, dec_hook=dec_hook)
-    cf = msgspec.json.decode((p / "config.json").read_bytes())
-    db = msgspec.json.decode((p / "db.json").read_bytes())
-    return msgspec.convert(cf | db, Config)
 
-# Load/initialize config file
+@modifies_config
-print(conffile, config_update(lambda c: c))
+def update_config(config: Config, changes: dict) -> Config:
+    """Create/update the config with new values, respecting changes done by others."""
+    # Encode into dict, update values with new, convert to Config
+    settings = {} if config is None else msgspec.to_builtins(config, enc_hook=enc_hook)
+    settings.update(changes)
+    return msgspec.convert(settings, Config, dec_hook=dec_hook)

cista/droppy.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+import msgspec
+
+
+def readconf() -> dict:
+    p = Path.home() / ".droppy/config"  # Hardcoded in Droppy
+    cf = msgspec.json.decode((p / "config.json").read_bytes())
+    db = msgspec.json.decode((p / "db.json").read_bytes())
+    cf["path"] = p.parent / "files"
+    cf["listen"] = _droppy_listeners(cf)
+    return cf | db
+
+def _droppy_listeners(cf):
+    """Convert Droppy listeners to our format, for typical cases but not in full."""
+    for listener in cf["listeners"]:
+        try:
+            if listener["protocol"] == "https":
+                # TODO: Add support for TLS
+                continue
+            socket = listener.get("socket")
+            if socket:
+                if isinstance(socket, list): socket = socket[0]
+                return f"{socket}"
+            port = listener["port"]
+            if isinstance(port, list): port = port[0]
+            host = listener["host"]
+            if isinstance(host, list): host = host[0]
+            if host in ("127.0.0.1", "::", "localhost"): return f":{port}"
+            return f"{host}:{port}"
+        except (KeyError, IndexError):
+            continue
+    # If none matched, fallback to Droppy default
+    return f"0.0.0.0:8989"

cista/fileio.py

@@ -9,8 +9,6 @@ from . import config
 from .asynclink import AsyncLink
 from .lrucache import LRUCache
 
-ROOT = config.config.path
-print("Serving", ROOT)
 
 def fuid(stat) -> str:
     """Unique file ID. Stays the same on renames and modification."""
@@ -26,7 +24,7 @@ def sanitize_filename(filename: str) -> str:
 
 class File:
     def __init__(self, filename):
-        self.path = ROOT / filename
+        self.path = config.config.path / filename
         self.fd = None
         self.writable = False
 

cista/httpredir.py

@@ -0,0 +1,21 @@
+from sanic import Sanic, exceptions, response
+
+app = Sanic("http_redirect")
+
+# Send all HTTP users to HTTPS
+@app.exception(exceptions.NotFound, exceptions.MethodNotSupported)
+def redirect_everything_else(request, exception):
+    server, path = request.server_name, request.path
+    if server and path.startswith("/"):
+        return response.redirect(f"https://{server}{path}", status=308)
+    return response.text("Bad Request. Please use HTTPS!", status=400)
+
+# ACME challenge for LetsEncrypt
+app.get("/.well-known/acme-challenge/<challenge>")
+async def letsencrypt(request, challenge):
+    try:
+        return response.text(acme_challenges[challenge])
+    except KeyError:
+        return response.text(f"ACME challenge not found: {challenge}", status=404)
+
+acme_challenges = {}

cista/serve.py

@@ -0,0 +1,37 @@
+import os
+import re
+from pathlib import Path
+
+from sanic import Sanic
+
+from . import config
+
+
+def run(dev=False):
+    """Run Sanic main process that spawns worker processes to serve HTTP requests."""
+    from .app import app
+    url, opts = parse_listen(config.config.listen)
+    # Silence Sanic's warning about running in production rather than debug
+    os.environ["SANIC_IGNORE_PRODUCTION_WARNING"] = "1"
+    if opts.get("ssl"):
+        # Run plain HTTP redirect/acme server on port 80
+        from . import httpredir
+        httpredir.app.prepare(port=80, motd=False)
+    app.prepare(**opts, motd=False, dev=dev, auto_reload=dev)
+    Sanic.serve()
+
+def parse_listen(listen):
+    if listen.startswith("/"):
+        unix = Path(listen).resolve()
+        if not unix.parent.exists():
+            raise ValueError(f"Directory for unix socket does not exist: {unix.parent}/")
+        return "http://localhost", {"unix": unix}
+    elif re.fullmatch(r"(\w+(-\w+)*\.)+\w{2,}", listen, re.UNICODE):
+        return f"https://{listen}", {"host": listen, "ssl": True}
+    else:
+        try:
+            addr, _port = listen.split(":", 1)
+            port = int(_port)
+        except Exception:
+            raise ValueError(f"Invalid listen address: {listen}")
+        return f"http://localhost:{port}", {"host": addr, "port": port}

cista/session.py

@@ -4,12 +4,12 @@ import jwt
 
 from .config import derived_secret
 
-session_secret = derived_secret("session")
+session_secret = lambda: derived_secret("session")
 max_age = 60  # Seconds since last login
 
 def get(request):
     try:
-        return jwt.decode(request.cookies.s, session_secret, algorithms=["HS256"])
+        return jwt.decode(request.cookies.s, session_secret(), algorithms=["HS256"])
     except Exception as e:
         s = None
         return False if "s" in request.cookies else None
@@ -20,12 +20,12 @@ def create(res, username, **kwargs):
         "username": username,
         **kwargs,
     }
-    s = jwt.encode(data, session_secret)
+    s = jwt.encode(data, session_secret())
     res.cookies.add_cookie("s", s, host_prefix=True, httponly=True, max_age=max_age)
 
 def update(res, s, **kwargs):
     s.update(kwargs)
-    s = jwt.encode(s, session_secret)
+    s = jwt.encode(s, session_secret())
     res.cookies.add_cookie("s", s, host_prefix=True, httponly=True, max_age=max(1, s["exp"] - int(time())))
 
 def delete(res):

cista/watching.py

@@ -8,13 +8,12 @@ from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
 from . import config
-from .fileio import ROOT
 from .protocol import DirEntry, FileEntry, UpdateEntry
 
 secret = secrets.token_bytes(8)
 pubsub = {}
 
-def walk(path: Path = ROOT) -> DirEntry | FileEntry | None:
+def walk(path: Path) -> DirEntry | FileEntry | None:
     try:
         s = path.stat()
         mtime = int(s.st_mtime)
@@ -34,8 +33,9 @@ def walk(path: Path = ROOT) -> DirEntry | FileEntry | None:
         print("OS error walking path", path, e)
         return None
 
-tree = {"": walk()}
+tree = None
 tree_lock = threading.Lock()
+rootpath = None
 
 def refresh():
     root = tree[""]
@@ -44,7 +44,7 @@ def refresh():
     ]}).decode()
 
 def update(relpath: PurePosixPath, loop):
-    new = walk(ROOT / relpath)
+    new = walk(rootpath / relpath)
     with tree_lock:
         msg = update_internal(relpath, new)
         print(msg)
@@ -102,11 +102,16 @@ async def broadcast(msg):
 def register(app, url):
     @app.before_server_start
     async def start_watcher(app, loop):
+        global tree, rootpath
+        config.load_config()
+        rootpath = config.config.path
+        # Pseudo nameless root entry to ease updates
+        tree = {"": walk(rootpath)}
         class Handler(FileSystemEventHandler):
             def on_any_event(self, event):
-                update(Path(event.src_path).relative_to(ROOT), loop)
+                update(Path(event.src_path).relative_to(rootpath), loop)
         app.ctx.observer = Observer()
-        app.ctx.observer.schedule(Handler(), str(ROOT), recursive=True)
+        app.ctx.observer.schedule(Handler(), str(rootpath), recursive=True)
         app.ctx.observer.start()
 
     @app.after_server_stop

pyproject.toml

@@ -9,12 +9,13 @@ description = "Dropbox-like file server with modern web interface"
 readme = "README.md"
 license = ""
 authors = [
-    { name = "Vasanko lda" },
+    { name = "Vasanko" },
 ]
 classifiers = [
 ]
 dependencies = [
     "argon2-cffi",
+    "docopt",
     "msgspec",
     "pathvalidate",
     "pyjwt",
@@ -26,8 +27,11 @@ dependencies = [
 [project.urls]
 Homepage = ""
 
-[tool.hatch]
+[project.scripts]
-version.source = "vcs"
+cista = "cista.__main__:main"
+
+[tool.hatch.version]
+source = "vcs"
 
 [tool.hatch.build]
 hooks.vcs.version-file = "cista/_version.py"

tests/test_lrucache.py

@@ -1,8 +1,11 @@
-import pytest
-from unittest.mock import Mock
 from time import sleep
+from unittest.mock import Mock
+
+import pytest
+
 from cista.lrucache import LRUCache  # Replace with actual import
 
+
 def mock_open(key):
     mock = Mock()
     mock.close = Mock()