Enforce 128 MiB minimum free space on uploads
- Add cista/util/diskspace.py with MIN_FREE_BYTES limit and cached check_free_space() helper. - Check available space in File.write() before ftruncate/write. - Catch ENOSPC in File.write and re-raise as InsufficientStorageError. - upload_file_chunk catches both proactive and ENOSPC errors and returns HTTP 507 Insufficient Storage. - Add tests for low-disk rejection and ENOSPC handling.
This commit is contained in:
+15
-2
@@ -1,9 +1,11 @@
|
|||||||
|
import errno
|
||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from cista import config
|
from cista import config
|
||||||
from cista.util import filename
|
from cista.util import filename
|
||||||
|
from cista.util.diskspace import InsufficientStorageError, check_free_space
|
||||||
from cista.util.lrucache import LRUCache
|
from cista.util.lrucache import LRUCache
|
||||||
|
|
||||||
|
|
||||||
@@ -34,13 +36,24 @@ class File:
|
|||||||
self.open_rw()
|
self.open_rw()
|
||||||
if self.fd is None:
|
if self.fd is None:
|
||||||
raise RuntimeError("file descriptor is not available for write")
|
raise RuntimeError("file descriptor is not available for write")
|
||||||
|
check_free_space(self.path)
|
||||||
if file_size is not None:
|
if file_size is not None:
|
||||||
if pos + len(buffer) > file_size:
|
if pos + len(buffer) > file_size:
|
||||||
raise ValueError("write exceeds declared file size")
|
raise ValueError("write exceeds declared file size")
|
||||||
os.ftruncate(self.fd, file_size)
|
try:
|
||||||
|
os.ftruncate(self.fd, file_size)
|
||||||
|
except OSError as e:
|
||||||
|
if e.errno == errno.ENOSPC:
|
||||||
|
raise InsufficientStorageError("No space left on device") from e
|
||||||
|
raise
|
||||||
if buffer:
|
if buffer:
|
||||||
os.lseek(self.fd, pos, os.SEEK_SET)
|
os.lseek(self.fd, pos, os.SEEK_SET)
|
||||||
os.write(self.fd, buffer)
|
try:
|
||||||
|
os.write(self.fd, buffer)
|
||||||
|
except OSError as e:
|
||||||
|
if e.errno == errno.ENOSPC:
|
||||||
|
raise InsufficientStorageError("No space left on device") from e
|
||||||
|
raise
|
||||||
|
|
||||||
def __getitem__(self, slc):
|
def __getitem__(self, slc):
|
||||||
if self.fd is None:
|
if self.fd is None:
|
||||||
|
|||||||
+19
-8
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import contextlib
|
import contextlib
|
||||||
|
import errno
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
@@ -12,11 +13,12 @@ from urllib.parse import unquote, urlparse
|
|||||||
from wsgiref.handlers import format_date_time
|
from wsgiref.handlers import format_date_time
|
||||||
|
|
||||||
from sanic import Blueprint, HTTPResponse, empty, json
|
from sanic import Blueprint, HTTPResponse, empty, json
|
||||||
from sanic.exceptions import BadRequest, NotFound
|
from sanic.exceptions import BadRequest, NotFound, SanicException
|
||||||
|
|
||||||
from cista import auth, config, sharefs, watching
|
from cista import auth, config, sharefs, watching
|
||||||
from cista.api import fileserver
|
from cista.api import fileserver
|
||||||
from cista.util import filename
|
from cista.util import filename
|
||||||
|
from cista.util.diskspace import InsufficientStorageError
|
||||||
|
|
||||||
bp = Blueprint("fileserver", url_prefix="/files")
|
bp = Blueprint("fileserver", url_prefix="/files")
|
||||||
|
|
||||||
@@ -52,13 +54,22 @@ async def upload_file_chunk(request, name):
|
|||||||
|
|
||||||
rel, path = _safe_relpath(name, request=request)
|
rel, path = _safe_relpath(name, request=request)
|
||||||
rel_name = rel.as_posix()
|
rel_name = rel.as_posix()
|
||||||
upload_info = await asyncio.to_thread(
|
try:
|
||||||
fileserver.upload_info,
|
upload_info = await asyncio.to_thread(
|
||||||
rel_name,
|
fileserver.upload_info,
|
||||||
start,
|
rel_name,
|
||||||
body,
|
start,
|
||||||
total,
|
body,
|
||||||
)
|
total,
|
||||||
|
)
|
||||||
|
except InsufficientStorageError as e:
|
||||||
|
raise SanicException(str(e), status_code=507, quiet=True) from e
|
||||||
|
except OSError as e:
|
||||||
|
if e.errno == errno.ENOSPC:
|
||||||
|
raise SanicException(
|
||||||
|
"No space left on device", status_code=507, quiet=True
|
||||||
|
) from e
|
||||||
|
raise
|
||||||
extras = []
|
extras = []
|
||||||
chunk_len = end - start
|
chunk_len = end - start
|
||||||
whole_file = start == 0 and end == total
|
whole_file = start == 0 and end == total
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import shutil
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
MIN_FREE_BYTES = 128 * 1024 * 1024
|
||||||
|
_CHECK_CACHE_TTL = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
class InsufficientStorageError(Exception):
|
||||||
|
"""Raised when there is not enough disk space for an operation."""
|
||||||
|
|
||||||
|
|
||||||
|
_cache: dict[Path, tuple[float, int]] = {}
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def check_free_space(path: Path) -> None:
|
||||||
|
"""Raise InsufficientStorageError if free space on the filesystem containing *path*
|
||||||
|
|
||||||
|
is below MIN_FREE_BYTES. Results are cached per directory for 1 second.
|
||||||
|
"""
|
||||||
|
check_path = path.parent if path.parent.exists() else path
|
||||||
|
check_path = check_path.resolve()
|
||||||
|
|
||||||
|
now = time.monotonic()
|
||||||
|
with _lock:
|
||||||
|
ts, free = _cache.get(check_path, (0, 0))
|
||||||
|
if now - ts < _CHECK_CACHE_TTL:
|
||||||
|
if free < MIN_FREE_BYTES:
|
||||||
|
raise InsufficientStorageError(
|
||||||
|
f"Insufficient storage: {free} bytes free, "
|
||||||
|
f"need at least {MIN_FREE_BYTES} bytes"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
free = shutil.disk_usage(check_path).free
|
||||||
|
except OSError as e:
|
||||||
|
raise InsufficientStorageError(f"Cannot check disk usage: {e}") from e
|
||||||
|
|
||||||
|
with _lock:
|
||||||
|
_cache[check_path] = (now, free)
|
||||||
|
|
||||||
|
if free < MIN_FREE_BYTES:
|
||||||
|
raise InsufficientStorageError(
|
||||||
|
f"Insufficient storage: {free} bytes free, "
|
||||||
|
f"need at least {MIN_FREE_BYTES} bytes"
|
||||||
|
)
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import errno
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import NamedTuple
|
||||||
|
from unittest.mock import patch
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from sanic import Sanic
|
||||||
|
|
||||||
|
from cista import config, watching
|
||||||
|
from cista.api import fileserver
|
||||||
|
from cista.fileserver import bp as fileserver_bp
|
||||||
|
|
||||||
|
|
||||||
|
class Usage(NamedTuple):
|
||||||
|
total: int
|
||||||
|
used: int
|
||||||
|
free: int
|
||||||
|
|
||||||
|
|
||||||
|
def _low_disk_usage(*args, **kwargs):
|
||||||
|
return Usage(total=1000, used=900, free=10)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def setup_storage(tmp_path: Path):
|
||||||
|
config.config = config.Config(path=tmp_path, listen=":0", public=True)
|
||||||
|
watching.state.root = []
|
||||||
|
watching.rootpath = tmp_path
|
||||||
|
yield tmp_path
|
||||||
|
watching.state.root = []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture()
|
||||||
|
async def client(setup_storage: Path):
|
||||||
|
app = Sanic(f"disk-space-test-{uuid4().hex}", strict_slashes=True)
|
||||||
|
app.router.ALLOWED_METHODS = (
|
||||||
|
*app.router.ALLOWED_METHODS,
|
||||||
|
"MKCOL",
|
||||||
|
"MOVE",
|
||||||
|
"COPY",
|
||||||
|
"PROPFIND",
|
||||||
|
)
|
||||||
|
app.blueprint(fileserver_bp)
|
||||||
|
await fileserver.start()
|
||||||
|
yield app.asgi_client
|
||||||
|
await fileserver.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_upload_rejected_when_disk_low(client):
|
||||||
|
with patch("cista.util.diskspace.shutil.disk_usage", side_effect=_low_disk_usage):
|
||||||
|
_, res = await client.put("/files/test.txt", data=b"hello world")
|
||||||
|
assert res.status_code == 507
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_upload_rejected_on_enospc(client):
|
||||||
|
with patch(
|
||||||
|
"cista.fileio.os.write",
|
||||||
|
side_effect=OSError(errno.ENOSPC, "No space left on device"),
|
||||||
|
):
|
||||||
|
_, res = await client.put("/files/test.txt", data=b"hello world")
|
||||||
|
assert res.status_code == 507
|
||||||
Reference in New Issue
Block a user