Implement database rotation with n days retention #2
+6
-4
@@ -44,13 +44,15 @@ The history that aged out is preserved at:
|
|||||||
- The timestamp is the **ts of the last record dropped by the rotation** (see
|
- The timestamp is the **ts of the last record dropped by the rotation** (see
|
||||||
step 4 — the leading snapshot of the rewritten main file carries the same
|
step 4 — the leading snapshot of the rewritten main file carries the same
|
||||||
ts), not the current time. The name tells you exactly which point in history
|
ts), not the current time. The name tells you exactly which point in history
|
||||||
the rotated file ends at. Use a filesystem-safe rendering (e.g.
|
the rotated file ends at. Rendered in ISO 8601 basic format with the same
|
||||||
`2026-09-02T15-24-57` — no `:` characters, which are awkward on some
|
microsecond precision as the record's ``ts`` in the database (e.g.
|
||||||
filesystems).
|
`20260902T143000.123456Z`), so the filename matches precisely the ``ts`` of
|
||||||
|
the final line of the rotated file and of the snapshot at the start of the
|
||||||
|
new file.
|
||||||
- The rotated name always ends in `.kantadb`, regardless of the original
|
- The rotated name always ends in `.kantadb`, regardless of the original
|
||||||
extension. Users may name their databases with no extension, `.kantadb`, or
|
extension. Users may name their databases with no extension, `.kantadb`, or
|
||||||
anything else (`.db`, …). Since the rotated name is derived from the *stem*,
|
anything else (`.db`, …). Since the rotated name is derived from the *stem*,
|
||||||
all of these work uniformly: `data` → `data@2026-09-02T15-24-57.kantadb`,
|
all of these work uniformly: `data` → `data@20260902T143000.123456Z.kantadb`,
|
||||||
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
|
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
|
||||||
- Rotated files live in the same directory.
|
- Rotated files live in the same directory.
|
||||||
- Collision: if a rotated file with the same name already exists (rotation
|
- Collision: if a rotated file with the same name already exists (rotation
|
||||||
|
|||||||
@@ -83,6 +83,10 @@ if sys.platform == "win32":
|
|||||||
]
|
]
|
||||||
_kernel32.CloseHandle.restype = wintypes.BOOL
|
_kernel32.CloseHandle.restype = wintypes.BOOL
|
||||||
_kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
_kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||||
|
_kernel32.SetEndOfFile.restype = wintypes.BOOL
|
||||||
|
_kernel32.SetEndOfFile.argtypes = [wintypes.HANDLE]
|
||||||
|
_kernel32.FlushFileBuffers.restype = wintypes.BOOL
|
||||||
|
_kernel32.FlushFileBuffers.argtypes = [wintypes.HANDLE]
|
||||||
|
|
||||||
def _is_invalid_handle(handle) -> bool:
|
def _is_invalid_handle(handle) -> bool:
|
||||||
return ctypes.c_void_p(handle).value == ctypes.c_void_p(-1).value
|
return ctypes.c_void_p(handle).value == ctypes.c_void_p(-1).value
|
||||||
@@ -177,6 +181,22 @@ class LockedFile:
|
|||||||
os.lseek(self._fd, current, os.SEEK_SET)
|
os.lseek(self._fd, current, os.SEEK_SET)
|
||||||
return end
|
return end
|
||||||
|
|
||||||
|
def replace_content(self, data: bytes) -> None:
|
||||||
|
"""Atomically-ish rewrite the file's content in place, lock retained.
|
||||||
|
|
||||||
|
Seeks to the start, truncates, writes *data* and fsyncs, all on the
|
||||||
|
already-locked descriptor. The path is never unlinked or renamed, so
|
||||||
|
no other process can observe a missing file or acquire its own lock.
|
||||||
|
Used by database rotation.
|
||||||
|
"""
|
||||||
|
if self._fd is None:
|
||||||
|
raise RuntimeError("LockedFile.replace_content() called on a closed file")
|
||||||
|
|
||||||
|
if sys.platform == "win32":
|
||||||
|
self._replace_content_win32(data)
|
||||||
|
else:
|
||||||
|
self._replace_content_unix(data)
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
"""Release the lock and close the file."""
|
"""Release the lock and close the file."""
|
||||||
if self._fd is None:
|
if self._fd is None:
|
||||||
@@ -227,6 +247,15 @@ class LockedFile:
|
|||||||
os.lseek(self._fd, 0, os.SEEK_END)
|
os.lseek(self._fd, 0, os.SEEK_END)
|
||||||
os.write(self._fd, data)
|
os.write(self._fd, data)
|
||||||
|
|
||||||
|
def _replace_content_unix(self, data: bytes) -> None:
|
||||||
|
os.lseek(self._fd, 0, os.SEEK_SET)
|
||||||
|
os.ftruncate(self._fd, 0)
|
||||||
|
view = memoryview(data)
|
||||||
|
while view:
|
||||||
|
written = os.write(self._fd, view)
|
||||||
|
view = view[written:]
|
||||||
|
os.fdatasync(self._fd)
|
||||||
|
|
||||||
# -- Windows -------------------------------------------------------------
|
# -- Windows -------------------------------------------------------------
|
||||||
|
|
||||||
def _open_win32(self, path: Path, create: bool, readonly: bool) -> None:
|
def _open_win32(self, path: Path, create: bool, readonly: bool) -> None:
|
||||||
@@ -288,3 +317,24 @@ class LockedFile:
|
|||||||
)
|
)
|
||||||
if not ok:
|
if not ok:
|
||||||
raise OSError(f"WriteFile failed: Windows error {ctypes.get_last_error()}")
|
raise OSError(f"WriteFile failed: Windows error {ctypes.get_last_error()}")
|
||||||
|
|
||||||
|
def _replace_content_win32(self, data: bytes) -> None:
|
||||||
|
_kernel32.SetFilePointer(self._fd, 0, None, _FILE_BEGIN)
|
||||||
|
written = wintypes.DWORD()
|
||||||
|
ok = _kernel32.WriteFile(
|
||||||
|
self._fd,
|
||||||
|
data,
|
||||||
|
len(data),
|
||||||
|
ctypes.byref(written),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not ok:
|
||||||
|
raise OSError(f"WriteFile failed: Windows error {ctypes.get_last_error()}")
|
||||||
|
if not _kernel32.SetEndOfFile(self._fd):
|
||||||
|
raise OSError(
|
||||||
|
f"SetEndOfFile failed: Windows error {ctypes.get_last_error()}"
|
||||||
|
)
|
||||||
|
if not _kernel32.FlushFileBuffers(self._fd):
|
||||||
|
raise OSError(
|
||||||
|
f"FlushFileBuffers failed: Windows error {ctypes.get_last_error()}"
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""Tests for LockedFile low-level behaviors."""
|
||||||
|
|
||||||
|
from kanta.exceptions import FileLockError
|
||||||
|
from kanta.filelock import LockedFile
|
||||||
|
|
||||||
|
|
||||||
|
def test_replace_content_rewrites_in_place(tmp_path):
|
||||||
|
path = tmp_path / "data.kantadb"
|
||||||
|
path.write_bytes(b"original content here")
|
||||||
|
|
||||||
|
f = LockedFile()
|
||||||
|
f.open(path)
|
||||||
|
try:
|
||||||
|
f.replace_content(b"new")
|
||||||
|
assert f.size() == 3
|
||||||
|
f.write(b"!")
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
assert path.read_bytes() == b"new!"
|
||||||
|
|
||||||
|
|
||||||
|
def test_replace_content_keeps_lock(tmp_path):
|
||||||
|
path = tmp_path / "data.kantadb"
|
||||||
|
path.write_bytes(b"abc")
|
||||||
|
|
||||||
|
f = LockedFile()
|
||||||
|
f.open(path)
|
||||||
|
try:
|
||||||
|
f.replace_content(b"xyz")
|
||||||
|
other = LockedFile()
|
||||||
|
try:
|
||||||
|
other.open(path)
|
||||||
|
raise AssertionError("second open should fail while lock is held")
|
||||||
|
except FileLockError:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_replace_content_grow_and_shrink(tmp_path):
|
||||||
|
path = tmp_path / "data.kantadb"
|
||||||
|
path.write_bytes(b"x" * 100)
|
||||||
|
|
||||||
|
f = LockedFile()
|
||||||
|
f.open(path)
|
||||||
|
try:
|
||||||
|
f.replace_content(b"")
|
||||||
|
assert f.size() == 0
|
||||||
|
f.replace_content(b"y" * 200)
|
||||||
|
assert f.size() == 200
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
assert path.read_bytes() == b"y" * 200
|
||||||
Reference in New Issue
Block a user