Replace jsondiff producer with own diff; support list edits in replay

compute_diff now emits a simple subset of jsondiff's marshaled syntax:
plain key assignment for adds and full-value changes (lists replaced
wholesale), $delete for removed object keys, and $replace only when a
dict replaces a non-dict. Removed keys are always $delete, even when the
object becomes empty (jsondiff emitted $replace: {}).

apply_diff (single implementation in serialization.base, used by both
replay and patch_state) additionally accepts jsondiff-produced patches:
positional $insert/$delete and per-index nested diffs on lists, plus
$ escaping of user keys and values per the marshal format.

jsondiff moves to the dev dependency group; it is used only by tests
that verify patch compatibility in both directions.
This commit is contained in:
2026-09-02 04:06:36 +00:00
parent bcf8ccb830
commit 379b0de1dc
7 changed files with 376 additions and 73 deletions
+55 -46
View File
@@ -1,63 +1,72 @@
"""Diff computation and replay utilities."""
"""Diff computation and replay utilities.
import jsondiff
Diffs use the jsondiff *marshal* format: JSON-serializable dicts where
command keys are ``$delete``, ``$insert`` and ``$replace``, and any user
string (key or value) starting with ``$`` is escaped by prepending another
``$`` (``$foo`` -> ``$$foo``).
Our producer is deliberately simpler than jsondiff's: only object keys are
added/removed recursively; every other change (scalars, lists, type
changes) is a plain full-value assignment. Removed keys are always emitted
as ``$delete``, even when the object becomes empty. The output stays a
subset of jsondiff's marshaled syntax, so ``jsondiff.patch(...,
marshal=True)`` can apply it, and our patcher can apply
jsondiff-produced diffs (including positional ``$insert``/``$delete``
list edits and per-index nested diffs).
"""
from kanta.structs import ChangeRecord
from kanta.serialization.base import ReplayResult, replay
from kanta.serialization.base import ReplayResult, apply_diff, marshal_value, replay
from kanta.serialization.framing import LineFramer
from kanta.serialization.json import JsonSerializer
_UNCHANGED = object()
def _escape_key(key: str) -> str:
"""Escape a user key for use as a diff key (``$foo`` -> ``$$foo``)."""
if isinstance(key, str) and key.startswith("$"):
return "$" + key
return key
def _diff(previous, current):
"""Compute a raw diff, or _UNCHANGED if there is no difference."""
if isinstance(previous, dict) and isinstance(current, dict):
result = {}
deleted = [_escape_key(k) for k in previous if k not in current]
if deleted:
result["$delete"] = deleted
for key, new_value in current.items():
ekey = _escape_key(key)
if key not in previous:
result[ekey] = marshal_value(new_value)
else:
sub = _diff(previous[key], new_value)
if sub is not _UNCHANGED:
result[ekey] = sub
return result if result else _UNCHANGED
if previous == current:
return _UNCHANGED
if isinstance(current, dict):
# A dict assigned over a non-dict is ambiguous with a nested diff;
# like jsondiff, use $replace for that case.
return {"$replace": marshal_value(current)}
return marshal_value(current)
def compute_diff(previous: dict, current: dict) -> dict | None:
"""Compute a jsondiff patch between two dicts.
"""Compute a marshaled diff between two state dicts.
Returns None if there is no difference.
"""
return jsondiff.diff(previous, current, marshal=True) or None
def _apply_diff(state: dict, diff: dict) -> dict:
"""Apply a jsondiff patch manually, handling ``$replace`` and ``$delete``.
jsondiff.patch does not handle nested ``$replace`` commands when the
parent key is missing from the state. This function recursively applies
diffs, treating ``$replace`` as full replacement and ``$delete`` as
key removal.
"""
if not isinstance(diff, dict):
return diff
result = dict(state) if isinstance(state, dict) else state
if not isinstance(result, dict):
result = {}
for key, value in diff.items():
if key == "$replace":
return value
elif key == "$delete":
if isinstance(value, list):
for k in value:
result.pop(k, None)
else:
result.pop(value, None)
elif isinstance(value, dict):
old = result.get(key, {})
if not isinstance(old, dict):
old = {}
result[key] = _apply_diff(old, value)
else:
result[key] = value
return result
diff = _diff(previous, current)
return diff if diff is not _UNCHANGED else None
def patch_state(state: dict, diff: dict) -> dict:
"""Apply a jsondiff patch to a state dict.
The diff was produced with ``marshal=True`` (string keys like
``"$replace"`` and ``"$delete"``) and decoded from JSON.
"""
return _apply_diff(state, diff)
"""Apply a marshaled diff to a state dict."""
return apply_diff(state, diff)
# Backward-compatible JSONL replay using the default serializer.
+17 -2
View File
@@ -15,6 +15,7 @@ from typing import Any
import msgspec
from kanta.serialization.base import _apply, unmarshal
from kanta.tty import Line, displaywidth
transaction_logger = logging.getLogger("kanta.transaction")
@@ -312,6 +313,13 @@ def _collect_changes(
changes.append(("update" if existed else "add", path, diff))
return
old_at_path = _get_nested(previous, path)
if isinstance(old_at_path, list):
# List edits ($insert/$delete/per-index) are shown as one whole-list
# update; the diff is already unmarshaled at this point.
changes.append(("update", path, _apply(old_at_path, diff)))
return
for key, value in diff.items():
if key == "$delete":
if isinstance(value, list):
@@ -340,7 +348,14 @@ def _collect_changes(
("update" if old_collection is not None else "add", path, value)
)
elif isinstance(key, str) and key.startswith("$"):
changes.append(("add", path, {key: value}))
# Unknown $-command or (post-unmarshal) a user key starting with
# "$": treat as a normal key.
new_path = path + [str(key)]
existed = _get_nested(previous, new_path) is not None
if existed:
_collect_changes(value, new_path, changes, previous)
else:
changes.append(("add", new_path, value))
else:
new_path = path + [str(key)]
existed = _get_nested(previous, new_path) is not None
@@ -418,7 +433,7 @@ def format_diff(
Returns a list of formatted lines (without newlines).
"""
changes: list[tuple[str, list[str], Any]] = []
_collect_changes(diff, [], changes, previous)
_collect_changes(unmarshal(diff), [], changes, previous)
if not changes:
return []
lines = []
+84 -20
View File
@@ -121,33 +121,97 @@ def replay(
def _patch_state(state: dict, diff: dict) -> dict:
return _apply_diff(state, diff)
return apply_diff(state, diff)
def _apply_diff(state: dict, diff: dict) -> dict:
if not isinstance(diff, dict):
_COMMAND_KEYS = frozenset({"$delete", "$insert", "$replace"})
def _escape(value: str) -> str:
"""Escape a user string for the marshaled format (``$x`` -> ``$$x``)."""
return "$" + value if value.startswith("$") else value
def _unescape(value: str) -> str:
"""Reverse escaping; command strings pass through unchanged."""
if value in _COMMAND_KEYS:
return value
if value.startswith("$"):
return value[1:]
return value
def marshal_value(value: Any) -> Any:
"""Escape all ``$``-prefixed strings in a value stored in a diff."""
if isinstance(value, dict):
return {
_escape(k) if isinstance(k, str) else k: marshal_value(v)
for k, v in value.items()
}
if isinstance(value, list):
return [marshal_value(v) for v in value]
if isinstance(value, str):
return _escape(value)
return value
def unmarshal(diff: Any) -> Any:
"""Unescape a marshaled diff (keys, values and ``$delete`` entries)."""
if isinstance(diff, dict):
return {
_unescape(k) if isinstance(k, str) else k: unmarshal(v)
for k, v in diff.items()
}
if isinstance(diff, list):
return [unmarshal(v) for v in diff]
if isinstance(diff, str):
return _unescape(diff)
return diff
result = dict(state) if isinstance(state, dict) else state
if not isinstance(result, dict):
result = {}
def apply_diff(state: Any, diff: Any) -> Any:
"""Apply a marshaled jsondiff-format diff.
Mirrors ``jsondiff.patch(..., marshal=True)``: supports ``$replace``,
key-based ``$delete`` on dicts, and positional ``$delete``/``$insert``
plus per-index nested diffs on lists.
"""
return _apply(state, unmarshal(diff))
def _apply(state: Any, diff: Any) -> Any:
if not isinstance(diff, dict):
return diff
if not diff:
return state
if "$replace" in diff:
return diff["$replace"]
if isinstance(state, list):
result = list(state)
deletes = diff.get("$delete")
if deletes:
for pos in deletes:
result.pop(pos)
for pos, value in diff.get("$insert", []):
result.insert(pos, value)
for key, value in diff.items():
if key == "$replace":
return value
if key == "$delete":
if isinstance(value, list):
for k in value:
result.pop(k, None)
else:
result.pop(value, None)
if key in ("$delete", "$insert"):
continue
if isinstance(value, dict):
old = result.get(key, {})
if not isinstance(old, dict):
old = {}
result[key] = _apply_diff(old, value)
pos = int(key)
result[pos] = _apply(result[pos], value)
return result
result = dict(state) if isinstance(state, dict) else {}
for key, value in diff.items():
if key == "$delete":
keys = value if isinstance(value, list) else [value]
for k in keys:
result.pop(k, None)
elif key == "$insert":
continue
elif key in result:
result[key] = _apply(result[key], value)
else:
result[key] = value
return result
+1 -1
View File
@@ -17,7 +17,6 @@ readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"blake3>=1.0.8",
"jsondiff>=2.2.1",
"msgspec>=0.20.0",
]
@@ -35,6 +34,7 @@ Repository = "https://git.zi.fi/LeoVasanko/kanta"
[dependency-groups]
dev = [
"jsondiff>=2.2.1",
"pytest>=9.0.2",
"pytest-asyncio>=1.3.0",
]
+215 -1
View File
@@ -1,8 +1,26 @@
from kanta.diff import compute_diff
"""Tests for our own diff producer/consumer and jsondiff compatibility.
jsondiff is a dev dependency used only here, to verify that:
- jsondiff.patch(..., marshal=True) can apply patches produced by
compute_diff (our format is a subset of jsondiff's marshaled syntax);
- apply_diff can apply patches produced by jsondiff.diff(..., marshal=True),
including positional $insert/$delete list edits and per-index nested diffs.
"""
import jsondiff
import pytest
from kanta.diff import compute_diff, patch_state
from kanta.logging import format_diff
from kanta.serialization.base import apply_diff
# --- Producer: compute_diff ------------------------------------------------
def test_no_diff():
assert compute_diff({"a": 1}, {"a": 1}) is None
assert compute_diff({}, {}) is None
def test_simple_diff():
@@ -14,3 +32,199 @@ def test_simple_diff():
def test_nested_diff():
diff = compute_diff({"x": {"y": 1}}, {"x": {"y": 2}})
assert diff == {"x": {"y": 2}}
def test_key_added():
assert compute_diff({"a": 1}, {"a": 1, "b": 2}) == {"b": 2}
def test_key_removed():
assert compute_diff({"a": 1, "b": 2}, {"a": 1}) == {"$delete": ["b"]}
def test_last_key_removed_is_delete_not_replace():
# jsondiff's minimal-diff search emits {"$replace": {}} here; we emit
# what actually happened: the key was deleted.
assert compute_diff({"a": 1}, {}) == {"$delete": ["a"]}
assert compute_diff({"x": {"y": 1}}, {"x": {}}) == {"x": {"$delete": ["y"]}}
def test_list_changes_are_full_assignment():
# No $insert/$delete positional edits: lists are replaced wholesale.
assert compute_diff({"l": [1, 2]}, {"l": [1, 2, 3]}) == {"l": [1, 2, 3]}
assert compute_diff({"l": [1, 2, 3]}, {"l": [1, 3]}) == {"l": [1, 3]}
assert compute_diff({"l": [1]}, {"l": []}) == {"l": []}
def test_list_with_unchanged_prefix_is_full_assignment():
diff = compute_diff({"l": ["a", "b", "c"]}, {"l": ["a", "x", "b", "c"]})
assert diff == {"l": ["a", "x", "b", "c"]}
def test_type_changes_are_full_assignment():
assert compute_diff({"a": {"x": 1}}, {"a": [1]}) == {"a": [1]}
# A dict replacing a non-dict needs $replace (a bare dict would read as
# a nested diff); this matches jsondiff.
assert compute_diff({"a": [1]}, {"a": {"x": 1}}) == {"a": {"$replace": {"x": 1}}}
assert compute_diff({"a": 1}, {"a": None}) == {"a": None}
def test_new_dict_value_assigned_wholesale():
assert compute_diff({}, {"a": {"x": 1}}) == {"a": {"x": 1}}
def test_dollar_keys_escaped():
assert compute_diff({}, {"$weird": 1}) == {"$$weird": 1}
assert compute_diff({"$weird": 1}, {"$weird": 2}) == {"$$weird": 2}
assert compute_diff({"$weird": 1}, {}) == {"$delete": ["$$weird"]}
def test_dollar_values_escaped():
assert compute_diff({"s": 1}, {"s": "$y"}) == {"s": "$$y"}
assert compute_diff({"s": 1}, {"s": "$delete"}) == {"s": "$$delete"}
# Nested values in wholesale assignments are escaped too.
assert compute_diff({}, {"o": {"s": "$y", "l": ["$z"]}}) == {
"o": {"s": "$$y", "l": ["$$z"]}
}
# --- Consumer: apply_diff / patch_state -------------------------------------
def test_patch_state_delegates():
assert patch_state({"a": 1}, {"a": 2}) == {"a": 2}
def test_apply_scalar_and_add():
assert apply_diff({"a": 1}, {"a": 2, "b": 3}) == {"a": 2, "b": 3}
def test_apply_delete():
assert apply_diff({"a": 1, "b": 2}, {"$delete": ["b"]}) == {"a": 1}
assert apply_diff({"a": 1}, {"$delete": ["a"]}) == {}
def test_apply_replace():
assert apply_diff({"a": 1, "b": 2}, {"$replace": {"c": 3}}) == {"c": 3}
assert apply_diff({"x": {"a": 1}}, {"x": {"$replace": [1]}}) == {"x": [1]}
def test_apply_nested_delete():
diff = {"x": {"$delete": ["y"]}}
assert apply_diff({"x": {"y": 2, "z": 3}}, diff) == {"x": {"z": 3}}
def test_apply_list_insert():
diff = {"l": {"$insert": [[1, "x"]]}}
assert apply_diff({"l": ["a", "b"]}, diff) == {"l": ["a", "x", "b"]}
def test_apply_list_delete():
diff = {"l": {"$delete": [1]}}
assert apply_diff({"l": ["a", "b", "c"]}, diff) == {"l": ["a", "c"]}
def test_apply_list_delete_multiple_positions():
# jsondiff emits positions in descending order for sequential pops.
diff = {"l": {"$delete": [4, 2, 0]}}
assert apply_diff({"l": [0, 1, 2, 3, 4]}, diff) == {"l": [1, 3]}
def test_apply_list_insert_and_delete():
diff = {"l": {"$insert": [[0, 9], [2, 8], [4, 7]], "$delete": [2, 0]}}
assert apply_diff({"l": [0, 1, 2, 3]}, diff) == {"l": [9, 1, 8, 3, 7]}
def test_apply_list_per_index_nested_diff():
diff = {"l": {"1": {"y": 3}}}
state = {"l": [{"x": 1}, {"y": 2}]}
assert apply_diff(state, diff) == {"l": [{"x": 1}, {"y": 3}]}
def test_apply_escaped_keys_and_values():
assert apply_diff({}, {"$$weird": 1}) == {"$weird": 1}
assert apply_diff({"$weird": 1}, {"$delete": ["$$weird"]}) == {}
assert apply_diff({"s": 1}, {"s": "$$y"}) == {"s": "$y"}
assert apply_diff({"s": 1}, {"s": "$$delete"}) == {"s": "$delete"}
assert apply_diff({}, {"o": {"s": "$$y", "l": ["$$z"]}}) == {
"o": {"s": "$y", "l": ["$z"]}
}
def test_apply_empty_diff():
assert apply_diff({"a": 1}, {}) == {"a": 1}
def test_apply_diff_on_missing_state():
assert apply_diff({}, {"a": {"b": 1}}) == {"a": {"b": 1}}
# --- jsondiff compatibility, both directions --------------------------------
COMPAT_CASES = [
("scalar change", {"a": 1}, {"a": 2}),
("key add", {"a": 1}, {"a": 1, "b": 2}),
("key remove", {"a": 1, "b": 2}, {"a": 1}),
("last key removed", {"a": 1}, {}),
("nested delete", {"a": {"x": 1, "y": 2}}, {"a": {"x": 1}}),
("nested mixed", {"a": {"x": 1, "y": 2}}, {"a": {"x": 9, "z": 3}}),
("list append", {"l": [1, 2]}, {"l": [1, 2, 3]}),
("list insert mid", {"l": [1, 2, 3]}, {"l": [1, 9, 2, 3]}),
("list remove mid", {"l": [1, 2, 3]}, {"l": [1, 3]}),
("list remove many", {"l": [0, 1, 2, 3, 4]}, {"l": [1, 3]}),
("list replace all", {"l": [1, 2]}, {"l": [3, 4]}),
("list insert+delete", {"l": [0, 1, 2, 3]}, {"l": [9, 1, 8, 3, 7]}),
("dict in list", {"l": [{"x": 1}, {"y": 2}]}, {"l": [{"x": 1}, {"y": 3}]}),
("list to empty", {"l": [1]}, {"l": []}),
("type change dict->list", {"a": {"x": 1}}, {"a": [1]}),
("type change list->dict", {"a": [1]}, {"a": {"x": 1}}),
("dollar key", {"$k": 1, "b": 1}, {"$k": 2}),
("dollar value", {"s": "$x"}, {"s": "$y"}),
(
"deep nesting",
{"a": {"b": {"c": {"d": 1, "e": 2}}}},
{"a": {"b": {"c": {"d": 9}}}},
),
]
@pytest.mark.parametrize("name,old,new", COMPAT_CASES, ids=[c[0] for c in COMPAT_CASES])
def test_jsondiff_applies_our_patches(name, old, new):
diff = compute_diff(old, new)
assert diff is not None
assert jsondiff.patch(old, diff, marshal=True) == new
@pytest.mark.parametrize("name,old,new", COMPAT_CASES, ids=[c[0] for c in COMPAT_CASES])
def test_we_apply_jsondiff_patches(name, old, new):
diff = jsondiff.diff(old, new, marshal=True)
assert apply_diff(old, diff) == new
@pytest.mark.parametrize("name,old,new", COMPAT_CASES, ids=[c[0] for c in COMPAT_CASES])
def test_our_own_round_trip(name, old, new):
diff = compute_diff(old, new)
assert diff is not None
assert apply_diff(old, diff) == new
def test_no_diff_means_equal_states():
for _name, old, new in COMPAT_CASES:
assert compute_diff(old, new) is not None # cases really differ
assert compute_diff({"a": [1, {"b": "$x"}]}, {"a": [1, {"b": "$x"}]}) is None
# --- Logging ----------------------------------------------------------------
def test_format_diff_list_edit_shows_whole_list():
diff = jsondiff.diff({"l": [1, 2, 3]}, {"l": [1, 9, 3]}, marshal=True)
lines = format_diff(diff, previous={"l": [1, 2, 3]})
text = "\n".join(lines)
assert "$insert" not in text
assert "[1, 9, 3]" in text
def test_format_diff_unescapes_dollar_keys():
lines = format_diff({"$$weird": 1}, previous={})
assert any("$weird" in line and "$$weird" not in line for line in lines)
+3 -2
View File
@@ -47,7 +47,7 @@ async def test_new_file_writes_bootstrap_record_without_handlers(
records = read_changes(path, format_config)
assert len(records) == 1
assert records[0].a == "bootstrap"
assert records[0].diff == {"$replace": {"users": {}, "counter": 0}}
assert records[0].diff == {"users": {}, "counter": 0}
@pytest.mark.asyncio
@@ -63,7 +63,8 @@ async def test_new_file_persists_initial_state_for_roundtrip(tmp_path, format_co
assert len(records) == 1
assert records[0].a == "bootstrap"
assert records[0].diff == {
"$replace": {"users": {"alice": {"name": "Alice", "age": 0}}, "counter": 5}
"users": {"alice": {"name": "Alice", "age": 0}},
"counter": 5,
}
kanta2 = make_kanta(path, Data, format_config)
+1 -1
View File
@@ -183,7 +183,7 @@ def test_apply_returns_change_information():
assert result.migrations[0].name == "migrate_v1"
assert result.migrations[0].description == "Set x"
assert result.migrations[0].changed is True
assert result.migrations[0].diff == {"$replace": {"x": 1}}
assert result.migrations[0].diff == {"x": 1}
assert result.migrations[1].name == "migrate_v2"
assert result.migrations[1].description == "No-op"