Implement database rotation with n days retention #2
+10
-8
@@ -44,15 +44,16 @@ The history that aged out is preserved at:
|
|||||||
- The timestamp is the **ts of the last record dropped by the rotation** (see
|
- The timestamp is the **ts of the last record dropped by the rotation** (see
|
||||||
step 4 — the leading snapshot of the rewritten main file carries the same
|
step 4 — the leading snapshot of the rewritten main file carries the same
|
||||||
ts), not the current time. The name tells you exactly which point in history
|
ts), not the current time. The name tells you exactly which point in history
|
||||||
the rotated file ends at. Rendered in ISO 8601 basic format with the same
|
the rotated file ends at. Rendered in ISO 8601 basic format at second
|
||||||
microsecond precision as the record's ``ts`` in the database (e.g.
|
precision (e.g. `20260902T143000Z`). The exact microsecond timestamp of the
|
||||||
`20260902T143000.123456Z`), so the filename matches precisely the ``ts`` of
|
cutoff remains available inside the file (it is the ``ts`` of the final
|
||||||
the final line of the rotated file and of the snapshot at the start of the
|
line of the rotated file and of the snapshot at the start of the new file);
|
||||||
new file.
|
a second rotation within the same second cannot occur because rotation
|
||||||
|
requires history to have aged past the cutoff.
|
||||||
- The rotated name always ends in `.kantadb`, regardless of the original
|
- The rotated name always ends in `.kantadb`, regardless of the original
|
||||||
extension. Users may name their databases with no extension, `.kantadb`, or
|
extension. Users may name their databases with no extension, `.kantadb`, or
|
||||||
anything else (`.db`, …). Since the rotated name is derived from the *stem*,
|
anything else (`.db`, …). Since the rotated name is derived from the *stem*,
|
||||||
all of these work uniformly: `data` → `data@20260902T143000.123456Z.kantadb`,
|
all of these work uniformly: `data` → `data@20260902T143000Z.kantadb`,
|
||||||
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
|
`data.kantadb` → `data@….kantadb`, `data.db` → `data@….kantadb`.
|
||||||
- Rotated files live in the same directory.
|
- Rotated files live in the same directory.
|
||||||
- Collision: if a rotated file with the same name already exists (rotation
|
- Collision: if a rotated file with the same name already exists (rotation
|
||||||
@@ -216,8 +217,9 @@ original `record_offset`, so binary checksums stay valid).
|
|||||||
Add keyword options to `Kanta(...)` (`kanta/kanta.py`), surfaced through
|
Add keyword options to `Kanta(...)` (`kanta/kanta.py`), surfaced through
|
||||||
`open()`:
|
`open()`:
|
||||||
|
|
||||||
- `retention: timedelta | None = None` — history window to keep. `None`
|
- `retention: timedelta | int | None = None` — history window to keep; a plain
|
||||||
(default) disables rotation entirely; current behavior is unchanged.
|
`int` is interpreted as a number of days. `None` (default) disables rotation
|
||||||
|
entirely; current behavior is unchanged.
|
||||||
- `rotate_keep: int = 3` (optional, later) — how many rotated backups to
|
- `rotate_keep: int = 3` (optional, later) — how many rotated backups to
|
||||||
retain; older ones are pruned at rotation time.
|
retain; older ones are pruned at rotation time.
|
||||||
|
|
||||||
|
|||||||
+8
-7
@@ -53,7 +53,7 @@ class Kanta(Generic[T]):
|
|||||||
migrations: ModuleType | str | None = None,
|
migrations: ModuleType | str | None = None,
|
||||||
serializer: Serializer | None = None,
|
serializer: Serializer | None = None,
|
||||||
flush_interval: float = 0.1,
|
flush_interval: float = 0.1,
|
||||||
retention: timedelta | None = None,
|
retention: timedelta | int | None = None,
|
||||||
):
|
):
|
||||||
"""Initialize a Kanta persistence instance.
|
"""Initialize a Kanta persistence instance.
|
||||||
|
|
||||||
@@ -64,12 +64,13 @@ class Kanta(Generic[T]):
|
|||||||
migrations: Optional migrations module object or import path.
|
migrations: Optional migrations module object or import path.
|
||||||
flush_interval: Background flush interval in seconds.
|
flush_interval: Background flush interval in seconds.
|
||||||
serializer: Optional serializer implementation.
|
serializer: Optional serializer implementation.
|
||||||
retention: Optional history retention window. When set, opening the
|
retention: Optional history retention window, either a
|
||||||
database rotates it: history older than ``now - retention`` is
|
:class:`~datetime.timedelta` or a plain number of days. When
|
||||||
moved to a ``{stem}@{timestamp}.kantadb`` sibling file and the
|
set, opening the database rotates it: history older than
|
||||||
main file is rewritten with a fresh snapshot plus the retained
|
``now - retention`` is moved to a ``{stem}@{timestamp}.kantadb``
|
||||||
records (see ``docs/rotation.md``). ``None`` (default) disables
|
sibling file and the main file is rewritten with a fresh
|
||||||
rotation.
|
snapshot plus the retained records (see ``docs/rotation.md``).
|
||||||
|
``None`` (default) disables rotation.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ImportError: If ``migrations`` is a string path that cannot be imported.
|
ImportError: If ``migrations`` is a string path that cannot be imported.
|
||||||
|
|||||||
+4
-1
@@ -43,7 +43,10 @@ class KantaImpl(PersistenceMixin, Generic[T]):
|
|||||||
self.data: T = kwargs.pop("data")
|
self.data: T = kwargs.pop("data")
|
||||||
self._kanta = kwargs.pop("kanta", None)
|
self._kanta = kwargs.pop("kanta", None)
|
||||||
migrations = kwargs.pop("migrations", None)
|
migrations = kwargs.pop("migrations", None)
|
||||||
self.retention: timedelta | None = kwargs.pop("retention", None)
|
retention = kwargs.pop("retention", None)
|
||||||
|
if isinstance(retention, int) and not isinstance(retention, bool):
|
||||||
|
retention = timedelta(days=retention)
|
||||||
|
self.retention: timedelta | None = retention
|
||||||
self.ctx = SimpleNamespace()
|
self.ctx = SimpleNamespace()
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self.migrations: Migrations | None = None
|
self.migrations: Migrations | None = None
|
||||||
|
|||||||
+5
-4
@@ -36,11 +36,12 @@ class RotationPlan:
|
|||||||
def rotated_path_for(path: Path, ts: datetime) -> Path:
|
def rotated_path_for(path: Path, ts: datetime) -> Path:
|
||||||
"""Sibling path for the rotated history: ``{stem}@{ISO-basic-ts}.kantadb``.
|
"""Sibling path for the rotated history: ``{stem}@{ISO-basic-ts}.kantadb``.
|
||||||
|
|
||||||
The timestamp uses ISO 8601 basic format with microsecond precision so it
|
The timestamp uses ISO 8601 basic format at second precision (e.g.
|
||||||
matches the record ``ts`` values stored in the database. On collision an
|
``20260902T143000Z``); the exact microsecond timestamp remains available
|
||||||
incrementing suffix is inserted before the extension.
|
inside the file if ever needed. On collision an incrementing suffix is
|
||||||
|
inserted before the extension.
|
||||||
"""
|
"""
|
||||||
stamp = ts.strftime("%Y%m%dT%H%M%S.%fZ")
|
stamp = ts.strftime("%Y%m%dT%H%M%SZ")
|
||||||
candidate = path.with_name(f"{path.stem}@{stamp}.kantadb")
|
candidate = path.with_name(f"{path.stem}@{stamp}.kantadb")
|
||||||
n = 1
|
n = 1
|
||||||
while candidate.exists():
|
while candidate.exists():
|
||||||
|
|||||||
+16
-2
@@ -77,7 +77,7 @@ async def test_rotation_splits_history(tmp_path, format_config):
|
|||||||
|
|
||||||
# Rotated file holds exactly the dropped history, ending at the last
|
# Rotated file holds exactly the dropped history, ending at the last
|
||||||
# dropped record whose ts matches the filename.
|
# dropped record whose ts matches the filename.
|
||||||
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ")
|
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
|
||||||
assert rotated[0].name == f"data@{stamp}.kantadb"
|
assert rotated[0].name == f"data@{stamp}.kantadb"
|
||||||
rrecords = read_all(rotated[0], format_config)
|
rrecords = read_all(rotated[0], format_config)
|
||||||
assert [r.a for r in rrecords] == ["bootstrap", "day-40"]
|
assert [r.a for r in rrecords] == ["bootstrap", "day-40"]
|
||||||
@@ -181,10 +181,24 @@ async def test_rotated_naming_normalizes_extension(tmp_path, format_config, name
|
|||||||
async with kanta:
|
async with kanta:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%S.%fZ")
|
stamp = (T0 - 40 * DAY).strftime("%Y%m%dT%H%M%SZ")
|
||||||
assert (tmp_path / f"data@{stamp}.kantadb").exists()
|
assert (tmp_path / f"data@{stamp}.kantadb").exists()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_retention_accepts_int_days(tmp_path, format_config):
|
||||||
|
path = tmp_path / "data.kantadb"
|
||||||
|
await write_history(path, format_config, days=[-40, -5])
|
||||||
|
|
||||||
|
cell = [T0]
|
||||||
|
kanta = make_kanta(path, Data, format_config, retention=30)
|
||||||
|
kanta.clock(make_clock(cell))
|
||||||
|
async with kanta:
|
||||||
|
assert kanta.data.counter == 2
|
||||||
|
|
||||||
|
assert len(rotated_files(path)) == 1
|
||||||
|
assert [c.a for c in read_changes(path, format_config)] == ["day-5"]
|
||||||
|
|
||||||
|
|
||||||
async def test_rotation_disabled_by_default(tmp_path, format_config):
|
async def test_rotation_disabled_by_default(tmp_path, format_config):
|
||||||
path = tmp_path / "data.kantadb"
|
path = tmp_path / "data.kantadb"
|
||||||
await write_history(path, format_config, days=[-40, -5])
|
await write_history(path, format_config, days=[-40, -5])
|
||||||
|
|||||||
Reference in New Issue
Block a user