Implement database rotation with n days retention #2
+10
-8
@@ -44,15 +44,16 @@ The history that aged out is preserved at:
|
||||
- The timestamp is the **ts of the last record dropped by the rotation** (see
|
||||
step 4 — the leading snapshot of the rewritten main file carries the same
|
||||
ts), not the current time. The name tells you exactly which point in history
|
||||
the rotated file ends at. Rendered in ISO 8601 basic format with the same
|
||||
microsecond precision as the record's ``ts`` in the database (e.g.
|
||||
`20260902T143000.123456Z`), so the filename matches precisely the ``ts`` of
|
||||
the final line of the rotated file and of the snapshot at the start of the
|
||||
new file.
|
||||
the rotated file ends at. Rendered in ISO 8601 basic format at second
|
||||
precision (e.g. `20260902T143000Z`). The exact microsecond timestamp of the
|
||||
cutoff remains available inside the file (it is the ``ts`` of the final
|
||||
line of the rotated file and of the snapshot at the start of the new file);
|
||||
a second rotation within the same second cannot occur because rotation
|
||||
requires history to have aged past the cutoff.
|
||||
- The rotated name always ends in `.kantadb`, regardless of the original
|
||||
extension. Users may name their databases with no extension, `.kantadb`, or
|
||||
anything else (`.db`, …). Since the rotated name is derived from the *stem*,
|
||||
all of these work uniformly: `data` → `data@20260902T143000.123456Z.kantadb`,
|
||||
all of these work uniformly: `data` → `data@20260902T143000Z.kantadb`,
|
||||
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
|
||||
- Rotated files live in the same directory.
|
||||
- Collision: if a rotated file with the same name already exists (rotation
|
||||
@@ -216,8 +217,9 @@ original `record_offset`, so binary checksums stay valid).
|
||||
Add keyword options to `Kanta(...)` (`kanta/kanta.py`), surfaced through
|
||||
`open()`:
|
||||
|
||||
- `retention: timedelta | None = None` — history window to keep. `None`
|
||||
(default) disables rotation entirely; current behavior is unchanged.
|
||||
- `retention: timedelta | int | None = None` — history window to keep; a plain
|
||||
`int` is interpreted as a number of days. `None` (default) disables rotation
|
||||
entirely; current behavior is unchanged.
|
||||
- `rotate_keep: int = 3` (optional, later) — how many rotated backups to
|
||||
retain; older ones are pruned at rotation time.
|
||||
|
||||
|
||||
+8
-7
@@ -53,7 +53,7 @@ class Kanta(Generic[T]):
|
||||
migrations: ModuleType | str | None = None,
|
||||
serializer: Serializer | None = None,
|
||||
flush_interval: float = 0.1,
|
||||
retention: timedelta | None = None,
|
||||
retention: timedelta | int | None = None,
|
||||
):
|
||||
"""Initialize a Kanta persistence instance.
|
||||
|
||||
@@ -64,12 +64,13 @@ class Kanta(Generic[T]):
|
||||
migrations: Optional migrations module object or import path.
|
||||
flush_interval: Background flush interval in seconds.
|
||||
serializer: Optional serializer implementation.
|
||||
retention: Optional history retention window. When set, opening the
|
||||
database rotates it: history older than ``now - retention`` is
|
||||
moved to a ``{stem}@{timestamp}.kantadb`` sibling file and the
|
||||
main file is rewritten with a fresh snapshot plus the retained
|
||||
records (see ``docs/rotation.md``). ``None`` (default) disables
|
||||
rotation.
|
||||
retention: Optional history retention window, either a
|
||||
:class:`~datetime.timedelta` or a plain number of days. When
|
||||
set, opening the database rotates it: history older than
|
||||
``now - retention`` is moved to a ``{stem}@{timestamp}.kantadb``
|
||||
sibling file and the main file is rewritten with a fresh
|
||||
snapshot plus the retained records (see ``docs/rotation.md``).
|
||||
``None`` (default) disables rotation.
|
||||
|
||||
Raises:
|
||||
ImportError: If ``migrations`` is a string path that cannot be imported.
|
||||
|
||||
+4
-1
@@ -43,7 +43,10 @@ class KantaImpl(PersistenceMixin, Generic[T]):
|
||||
self.data: T = kwargs.pop("data")
|
||||
self._kanta = kwargs.pop("kanta", None)
|
||||
migrations = kwargs.pop("migrations", None)
|
||||
self.retention: timedelta | None = kwargs.pop("retention", None)
|
||||
retention = kwargs.pop("retention", None)
|
||||
if isinstance(retention, int) and not isinstance(retention, bool):
|
||||
retention = timedelta(days=retention)
|
||||
self.retention: timedelta | None = retention
|
||||
self.ctx = SimpleNamespace()
|
||||
super().__init__(**kwargs)
|
||||
self.migrations: Migrations | None = None
|
||||
|
||||
+5
-4
@@ -36,11 +36,12 @@ class RotationPlan:
|
||||
def rotated_path_for(path: Path, ts: datetime) -> Path:
|
||||
"""Sibling path for the rotated history: ``{stem}@{ISO-basic-ts}.kantadb``.
|
||||
|
||||
The timestamp uses ISO 8601 basic format with microsecond precision so it
|
||||
matches the record ``ts`` values stored in the database. On collision an
|
||||
incrementing suffix is inserted before the extension.
|
||||
The timestamp uses ISO 8601 basic format at second precision (e.g.
|
||||
``20260902T143000Z``); the exact microsecond timestamp remains available
|
||||
inside the file if ever needed. On collision an incrementing suffix is
|
||||
inserted before the extension.
|
||||
"""
|
||||
stamp = ts.strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
stamp = ts.strftime("%Y%m%dT%H%M%SZ")
|
||||
candidate = path.with_name(f"{path.stem}@{stamp}.kantadb")
|
||||
n = 1
|
||||
while candidate.exists():
|
||||
|
||||
+16
-2
@@ -77,7 +77,7 @@ async def test_rotation_splits_history(tmp_path, format_config):
|
||||
|
||||
# Rotated file holds exactly the dropped history, ending at the last
|
||||
# dropped record whose ts matches the filename.
|
||||
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
|
||||
assert rotated[0].name == f"data@{stamp}.kantadb"
|
||||
rrecords = read_all(rotated[0], format_config)
|
||||
assert [r.a for r in rrecords] == ["bootstrap", "day-40"]
|
||||
@@ -181,10 +181,24 @@ async def test_rotated_naming_normalizes_extension(tmp_path, format_config, name
|
||||
async with kanta:
|
||||
pass
|
||||
|
||||
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
|
||||
assert (tmp_path / f"data@{stamp}.kantadb").exists()
|
||||
|
||||
|
||||
async def test_retention_accepts_int_days(tmp_path, format_config):
|
||||
path = tmp_path / "data.kantadb"
|
||||
await write_history(path, format_config, days=[-40, -5])
|
||||
|
||||
cell = [T0]
|
||||
kanta = make_kanta(path, Data, format_config, retention=30)
|
||||
kanta.clock(make_clock(cell))
|
||||
async with kanta:
|
||||
assert kanta.data.counter == 2
|
||||
|
||||
assert len(rotated_files(path)) == 1
|
||||
assert [c.a for c in read_changes(path, format_config)] == ["day-5"]
|
||||
|
||||
|
||||
async def test_rotation_disabled_by_default(tmp_path, format_config):
|
||||
path = tmp_path / "data.kantadb"
|
||||
await write_history(path, format_config, days=[-40, -5])
|
||||
|
||||
Reference in New Issue
Block a user