Implement database rotation with n days retention #2

Merged
LeoVasanko merged 8 commits from rotation into main 2026-09-02 16:46:19 +00:00
5 changed files with 43 additions and 22 deletions
Showing only changes of commit 3074de2950 - Show all commits
+10 -8
View File
@@ -44,15 +44,16 @@ The history that aged out is preserved at:
- The timestamp is the **ts of the last record dropped by the rotation** (see - The timestamp is the **ts of the last record dropped by the rotation** (see
step 4 — the leading snapshot of the rewritten main file carries the same step 4 — the leading snapshot of the rewritten main file carries the same
ts), not the current time. The name tells you exactly which point in history ts), not the current time. The name tells you exactly which point in history
the rotated file ends at. Rendered in ISO 8601 basic format with the same the rotated file ends at. Rendered in ISO 8601 basic format at second
microsecond precision as the record's ``ts`` in the database (e.g. precision (e.g. `20260902T143000Z`). The exact microsecond timestamp of the
`20260902T143000.123456Z`), so the filename matches precisely the ``ts`` of cutoff remains available inside the file (it is the ``ts`` of the final
the final line of the rotated file and of the snapshot at the start of the line of the rotated file and of the snapshot at the start of the new file);
new file. a second rotation within the same second cannot occur because rotation
requires history to have aged past the cutoff.
- The rotated name always ends in `.kantadb`, regardless of the original - The rotated name always ends in `.kantadb`, regardless of the original
extension. Users may name their databases with no extension, `.kantadb`, or extension. Users may name their databases with no extension, `.kantadb`, or
anything else (`.db`, …). Since the rotated name is derived from the *stem*, anything else (`.db`, …). Since the rotated name is derived from the *stem*,
all of these work uniformly: `data` → `data@20260902T143000.123456Z.kantadb`, all of these work uniformly: `data` → `data@20260902T143000Z.kantadb`,
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`. `data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
- Rotated files live in the same directory. - Rotated files live in the same directory.
- Collision: if a rotated file with the same name already exists (rotation - Collision: if a rotated file with the same name already exists (rotation
@@ -216,8 +217,9 @@ original `record_offset`, so binary checksums stay valid).
Add keyword options to `Kanta(...)` (`kanta/kanta.py`), surfaced through Add keyword options to `Kanta(...)` (`kanta/kanta.py`), surfaced through
`open()`: `open()`:
- `retention: timedelta | None = None` — history window to keep. `None` - `retention: timedelta | int | None = None` — history window to keep; a plain
(default) disables rotation entirely; current behavior is unchanged. `int` is interpreted as a number of days. `None` (default) disables rotation
entirely; current behavior is unchanged.
- `rotate_keep: int = 3` (optional, later) — how many rotated backups to - `rotate_keep: int = 3` (optional, later) — how many rotated backups to
retain; older ones are pruned at rotation time. retain; older ones are pruned at rotation time.
+8 -7
View File
@@ -53,7 +53,7 @@ class Kanta(Generic[T]):
migrations: ModuleType | str | None = None, migrations: ModuleType | str | None = None,
serializer: Serializer | None = None, serializer: Serializer | None = None,
flush_interval: float = 0.1, flush_interval: float = 0.1,
retention: timedelta | None = None, retention: timedelta | int | None = None,
): ):
"""Initialize a Kanta persistence instance. """Initialize a Kanta persistence instance.
@@ -64,12 +64,13 @@ class Kanta(Generic[T]):
migrations: Optional migrations module object or import path. migrations: Optional migrations module object or import path.
flush_interval: Background flush interval in seconds. flush_interval: Background flush interval in seconds.
serializer: Optional serializer implementation. serializer: Optional serializer implementation.
retention: Optional history retention window. When set, opening the retention: Optional history retention window, either a
database rotates it: history older than ``now - retention`` is :class:`~datetime.timedelta` or a plain number of days. When
moved to a ``{stem}@{timestamp}.kantadb`` sibling file and the set, opening the database rotates it: history older than
main file is rewritten with a fresh snapshot plus the retained ``now - retention`` is moved to a ``{stem}@{timestamp}.kantadb``
records (see ``docs/rotation.md``). ``None`` (default) disables sibling file and the main file is rewritten with a fresh
rotation. snapshot plus the retained records (see ``docs/rotation.md``).
``None`` (default) disables rotation.
Raises: Raises:
ImportError: If ``migrations`` is a string path that cannot be imported. ImportError: If ``migrations`` is a string path that cannot be imported.
+4 -1
View File
@@ -43,7 +43,10 @@ class KantaImpl(PersistenceMixin, Generic[T]):
self.data: T = kwargs.pop("data") self.data: T = kwargs.pop("data")
self._kanta = kwargs.pop("kanta", None) self._kanta = kwargs.pop("kanta", None)
migrations = kwargs.pop("migrations", None) migrations = kwargs.pop("migrations", None)
self.retention: timedelta | None = kwargs.pop("retention", None) retention = kwargs.pop("retention", None)
if isinstance(retention, int) and not isinstance(retention, bool):
retention = timedelta(days=retention)
self.retention: timedelta | None = retention
self.ctx = SimpleNamespace() self.ctx = SimpleNamespace()
super().__init__(**kwargs) super().__init__(**kwargs)
self.migrations: Migrations | None = None self.migrations: Migrations | None = None
+5 -4
View File
@@ -36,11 +36,12 @@ class RotationPlan:
def rotated_path_for(path: Path, ts: datetime) -> Path: def rotated_path_for(path: Path, ts: datetime) -> Path:
"""Sibling path for the rotated history: ``{stem}@{ISO-basic-ts}.kantadb``. """Sibling path for the rotated history: ``{stem}@{ISO-basic-ts}.kantadb``.
The timestamp uses ISO 8601 basic format with microsecond precision so it The timestamp uses ISO 8601 basic format at second precision (e.g.
matches the record ``ts`` values stored in the database. On collision an ``20260902T143000Z``); the exact microsecond timestamp remains available
incrementing suffix is inserted before the extension. inside the file if ever needed. On collision an incrementing suffix is
inserted before the extension.
""" """
stamp = ts.strftime("%Y%m%dT%H%M%S.%fZ") stamp = ts.strftime("%Y%m%dT%H%M%SZ")
candidate = path.with_name(f"{path.stem}@{stamp}.kantadb") candidate = path.with_name(f"{path.stem}@{stamp}.kantadb")
n = 1 n = 1
while candidate.exists(): while candidate.exists():
+16 -2
View File
@@ -77,7 +77,7 @@ async def test_rotation_splits_history(tmp_path, format_config):
# Rotated file holds exactly the dropped history, ending at the last # Rotated file holds exactly the dropped history, ending at the last
# dropped record whose ts matches the filename. # dropped record whose ts matches the filename.
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ") stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
assert rotated[0].name == f"data@{stamp}.kantadb" assert rotated[0].name == f"data@{stamp}.kantadb"
rrecords = read_all(rotated[0], format_config) rrecords = read_all(rotated[0], format_config)
assert [r.a for r in rrecords] == ["bootstrap", "day-40"] assert [r.a for r in rrecords] == ["bootstrap", "day-40"]
@@ -181,10 +181,24 @@ async def test_rotated_naming_normalizes_extension(tmp_path, format_config, name
async with kanta: async with kanta:
pass pass
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ") stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
assert (tmp_path / f"data@{stamp}.kantadb").exists() assert (tmp_path / f"data@{stamp}.kantadb").exists()
async def test_retention_accepts_int_days(tmp_path, format_config):
path = tmp_path / "data.kantadb"
await write_history(path, format_config, days=[-40, -5])
cell = [T0]
kanta = make_kanta(path, Data, format_config, retention=30)
kanta.clock(make_clock(cell))
async with kanta:
assert kanta.data.counter == 2
assert len(rotated_files(path)) == 1
assert [c.a for c in read_changes(path, format_config)] == ["day-5"]
async def test_rotation_disabled_by_default(tmp_path, format_config): async def test_rotation_disabled_by_default(tmp_path, format_config):
path = tmp_path / "data.kantadb" path = tmp_path / "data.kantadb"
await write_history(path, format_config, days=[-40, -5]) await write_history(path, format_config, days=[-40, -5])