Implement database rotation with n days retention #2

Merged
LeoVasanko merged 8 commits from rotation into main 2026-09-02 16:46:19 +00:00
3 changed files with 111 additions and 4 deletions
Showing only changes of commit 010b690b47 - Show all commits
+6 -4
View File
@@ -44,13 +44,15 @@ The history that aged out is preserved at:
- The timestamp is the **ts of the last record dropped by the rotation** (see - The timestamp is the **ts of the last record dropped by the rotation** (see
step 4 — the leading snapshot of the rewritten main file carries the same step 4 — the leading snapshot of the rewritten main file carries the same
ts), not the current time. The name tells you exactly which point in history ts), not the current time. The name tells you exactly which point in history
the rotated file ends at. Use a filesystem-safe rendering (e.g. the rotated file ends at. Rendered in ISO 8601 basic format with the same
`2026-09-02T15-24-57` — no `:` characters, which are awkward on some microsecond precision as the record's ``ts`` in the database (e.g.
filesystems). `20260902T143000.123456Z`), so the filename matches precisely the ``ts`` of
the final line of the rotated file and of the snapshot at the start of the
new file.
- The rotated name always ends in `.kantadb`, regardless of the original - The rotated name always ends in `.kantadb`, regardless of the original
extension. Users may name their databases with no extension, `.kantadb`, or extension. Users may name their databases with no extension, `.kantadb`, or
anything else (`.db`, …). Since the rotated name is derived from the *stem*, anything else (`.db`, …). Since the rotated name is derived from the *stem*,
all of these work uniformly: `data``data@2026-09-02T15-24-57.kantadb`, all of these work uniformly: `data` → `data@20260902T143000.123456Z.kantadb`,
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`. `data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
- Rotated files live in the same directory. - Rotated files live in the same directory.
- Collision: if a rotated file with the same name already exists (rotation - Collision: if a rotated file with the same name already exists (rotation
+50
View File
@@ -83,6 +83,10 @@ if sys.platform == "win32":
] ]
_kernel32.CloseHandle.restype = wintypes.BOOL _kernel32.CloseHandle.restype = wintypes.BOOL
_kernel32.CloseHandle.argtypes = [wintypes.HANDLE] _kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
_kernel32.SetEndOfFile.restype = wintypes.BOOL
_kernel32.SetEndOfFile.argtypes = [wintypes.HANDLE]
_kernel32.FlushFileBuffers.restype = wintypes.BOOL
_kernel32.FlushFileBuffers.argtypes = [wintypes.HANDLE]
def _is_invalid_handle(handle) -> bool: def _is_invalid_handle(handle) -> bool:
return ctypes.c_void_p(handle).value == ctypes.c_void_p(-1).value return ctypes.c_void_p(handle).value == ctypes.c_void_p(-1).value
@@ -177,6 +181,22 @@ class LockedFile:
os.lseek(self._fd, current, os.SEEK_SET) os.lseek(self._fd, current, os.SEEK_SET)
return end return end
def replace_content(self, data: bytes) -> None:
"""Atomically-ish rewrite the file's content in place, lock retained.
Seeks to the start, truncates, writes *data* and fsyncs, all on the
already-locked descriptor. The path is never unlinked or renamed, so
no other process can observe a missing file or acquire its own lock.
Used by database rotation.
"""
if self._fd is None:
raise RuntimeError("LockedFile.replace_content() called on a closed file")
if sys.platform == "win32":
self._replace_content_win32(data)
else:
self._replace_content_unix(data)
def close(self) -> None: def close(self) -> None:
"""Release the lock and close the file.""" """Release the lock and close the file."""
if self._fd is None: if self._fd is None:
@@ -227,6 +247,15 @@ class LockedFile:
os.lseek(self._fd, 0, os.SEEK_END) os.lseek(self._fd, 0, os.SEEK_END)
os.write(self._fd, data) os.write(self._fd, data)
def _replace_content_unix(self, data: bytes) -> None:
os.lseek(self._fd, 0, os.SEEK_SET)
os.ftruncate(self._fd, 0)
view = memoryview(data)
while view:
written = os.write(self._fd, view)
view = view[written:]
os.fdatasync(self._fd)
# -- Windows ------------------------------------------------------------- # -- Windows -------------------------------------------------------------
def _open_win32(self, path: Path, create: bool, readonly: bool) -> None: def _open_win32(self, path: Path, create: bool, readonly: bool) -> None:
@@ -288,3 +317,24 @@ class LockedFile:
) )
if not ok: if not ok:
raise OSError(f"WriteFile failed: Windows error {ctypes.get_last_error()}") raise OSError(f"WriteFile failed: Windows error {ctypes.get_last_error()}")
def _replace_content_win32(self, data: bytes) -> None:
_kernel32.SetFilePointer(self._fd, 0, None, _FILE_BEGIN)
written = wintypes.DWORD()
ok = _kernel32.WriteFile(
self._fd,
data,
len(data),
ctypes.byref(written),
None,
)
if not ok:
raise OSError(f"WriteFile failed: Windows error {ctypes.get_last_error()}")
if not _kernel32.SetEndOfFile(self._fd):
raise OSError(
f"SetEndOfFile failed: Windows error {ctypes.get_last_error()}"
)
if not _kernel32.FlushFileBuffers(self._fd):
raise OSError(
f"FlushFileBuffers failed: Windows error {ctypes.get_last_error()}"
)
+55
View File
@@ -0,0 +1,55 @@
"""Tests for LockedFile low-level behaviors."""
from kanta.exceptions import FileLockError
from kanta.filelock import LockedFile
def test_replace_content_rewrites_in_place(tmp_path):
path = tmp_path / "data.kantadb"
path.write_bytes(b"original content here")
f = LockedFile()
f.open(path)
try:
f.replace_content(b"new")
assert f.size() == 3
f.write(b"!")
finally:
f.close()
assert path.read_bytes() == b"new!"
def test_replace_content_keeps_lock(tmp_path):
path = tmp_path / "data.kantadb"
path.write_bytes(b"abc")
f = LockedFile()
f.open(path)
try:
f.replace_content(b"xyz")
other = LockedFile()
try:
other.open(path)
raise AssertionError("second open should fail while lock is held")
except FileLockError:
pass
finally:
f.close()
def test_replace_content_grow_and_shrink(tmp_path):
path = tmp_path / "data.kantadb"
path.write_bytes(b"x" * 100)
f = LockedFile()
f.open(path)
try:
f.replace_content(b"")
assert f.size() == 0
f.replace_content(b"y" * 200)
assert f.size() == 200
finally:
f.close()
assert path.read_bytes() == b"y" * 200