Implement database rotation with n days retention #2

Merged
LeoVasanko merged 8 commits from rotation into main 2026-09-02 16:46:19 +00:00
5 changed files with 43 additions and 22 deletions
Showing only changes of commit 3074de2950 - Show all commits
+10 -8
View File
@@ -44,15 +44,16 @@ The history that aged out is preserved at:
- The timestamp is the **ts of the last record dropped by the rotation** (see
step 4 — the leading snapshot of the rewritten main file carries the same
ts), not the current time. The name tells you exactly which point in history
the rotated file ends at. Rendered in ISO 8601 basic format with the same
microsecond precision as the record's ``ts`` in the database (e.g.
`20260902T143000.123456Z`), so the filename matches precisely the ``ts`` of
the final line of the rotated file and of the snapshot at the start of the
new file.
the rotated file ends at. Rendered in ISO 8601 basic format at second
precision (e.g. `20260902T143000Z`). The exact microsecond timestamp of the
cutoff remains available inside the file (it is the ``ts`` of the final
line of the rotated file and of the snapshot at the start of the new file);
a second rotation within the same second cannot occur because rotation
requires history to have aged past the cutoff.
- The rotated name always ends in `.kantadb`, regardless of the original
extension. Users may name their databases with no extension, `.kantadb`, or
anything else (`.db`, …). Since the rotated name is derived from the *stem*,
all of these work uniformly: `data` → `data@20260902T143000.123456Z.kantadb`,
all of these work uniformly: `data` → `data@20260902T143000Z.kantadb`,
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
- Rotated files live in the same directory.
- Collision: if a rotated file with the same name already exists (rotation
@@ -216,8 +217,9 @@ original `record_offset`, so binary checksums stay valid).
Add keyword options to `Kanta(...)` (`kanta/kanta.py`), surfaced through
`open()`:
- `retention: timedelta | None = None` — history window to keep. `None`
(default) disables rotation entirely; current behavior is unchanged.
- `retention: timedelta | int | None = None` — history window to keep; a plain
`int` is interpreted as a number of days. `None` (default) disables rotation
entirely; current behavior is unchanged.
- `rotate_keep: int = 3` (optional, later) — how many rotated backups to
retain; older ones are pruned at rotation time.
+8 -7
View File
@@ -53,7 +53,7 @@ class Kanta(Generic[T]):
migrations: ModuleType | str | None = None,
serializer: Serializer | None = None,
flush_interval: float = 0.1,
retention: timedelta | None = None,
retention: timedelta | int | None = None,
):
"""Initialize a Kanta persistence instance.
@@ -64,12 +64,13 @@ class Kanta(Generic[T]):
migrations: Optional migrations module object or import path.
flush_interval: Background flush interval in seconds.
serializer: Optional serializer implementation.
retention: Optional history retention window. When set, opening the
database rotates it: history older than ``now - retention`` is
moved to a ``{stem}@{timestamp}.kantadb`` sibling file and the
main file is rewritten with a fresh snapshot plus the retained
records (see ``docs/rotation.md``). ``None`` (default) disables
rotation.
retention: Optional history retention window, either a
:class:`~datetime.timedelta` or a plain number of days. When
set, opening the database rotates it: history older than
``now - retention`` is moved to a ``{stem}@{timestamp}.kantadb``
sibling file and the main file is rewritten with a fresh
snapshot plus the retained records (see ``docs/rotation.md``).
``None`` (default) disables rotation.
Raises:
ImportError: If ``migrations`` is a string path that cannot be imported.
+4 -1
View File
@@ -43,7 +43,10 @@ class KantaImpl(PersistenceMixin, Generic[T]):
self.data: T = kwargs.pop("data")
self._kanta = kwargs.pop("kanta", None)
migrations = kwargs.pop("migrations", None)
self.retention: timedelta | None = kwargs.pop("retention", None)
retention = kwargs.pop("retention", None)
if isinstance(retention, int) and not isinstance(retention, bool):
retention = timedelta(days=retention)
self.retention: timedelta | None = retention
self.ctx = SimpleNamespace()
super().__init__(**kwargs)
self.migrations: Migrations | None = None
+5 -4
View File
@@ -36,11 +36,12 @@ class RotationPlan:
def rotated_path_for(path: Path, ts: datetime) -> Path:
"""Sibling path for the rotated history: ``{stem}@{ISO-basic-ts}.kantadb``.
The timestamp uses ISO 8601 basic format with microsecond precision so it
matches the record ``ts`` values stored in the database. On collision an
incrementing suffix is inserted before the extension.
The timestamp uses ISO 8601 basic format at second precision (e.g.
``20260902T143000Z``); the exact microsecond timestamp remains available
inside the file if ever needed. On collision an incrementing suffix is
inserted before the extension.
"""
stamp = ts.strftime("%Y%m%dT%H%M%S.%fZ")
stamp = ts.strftime("%Y%m%dT%H%M%SZ")
candidate = path.with_name(f"{path.stem}@{stamp}.kantadb")
n = 1
while candidate.exists():
+16 -2
View File
@@ -77,7 +77,7 @@ async def test_rotation_splits_history(tmp_path, format_config):
# Rotated file holds exactly the dropped history, ending at the last
# dropped record whose ts matches the filename.
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ")
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
assert rotated[0].name == f"data@{stamp}.kantadb"
rrecords = read_all(rotated[0], format_config)
assert [r.a for r in rrecords] == ["bootstrap", "day-40"]
@@ -181,10 +181,24 @@ async def test_rotated_naming_normalizes_extension(tmp_path, format_config, name
async with kanta:
pass
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ")
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
assert (tmp_path / f"data@{stamp}.kantadb").exists()
async def test_retention_accepts_int_days(tmp_path, format_config):
path = tmp_path / "data.kantadb"
await write_history(path, format_config, days=[-40, -5])
cell = [T0]
kanta = make_kanta(path, Data, format_config, retention=30)
kanta.clock(make_clock(cell))
async with kanta:
assert kanta.data.counter == 2
assert len(rotated_files(path)) == 1
assert [c.a for c in read_changes(path, format_config)] == ["day-5"]
async def test_rotation_disabled_by_default(tmp_path, format_config):
path = tmp_path / "data.kantadb"
await write_history(path, format_config, days=[-40, -5])