diff --git a/kanta/diff.py b/kanta/diff.py index 725ab70..1f4db0d 100644 --- a/kanta/diff.py +++ b/kanta/diff.py @@ -1,63 +1,72 @@ -"""Diff computation and replay utilities.""" +"""Diff computation and replay utilities. -import jsondiff +Diffs use the jsondiff *marshal* format: JSON-serializable dicts where +command keys are ``$delete``, ``$insert`` and ``$replace``, and any user +string (key or value) starting with ``$`` is escaped by prepending another +``$`` (``$foo`` -> ``$$foo``). + +Our producer is deliberately simpler than jsondiff's: only object keys are +added/removed recursively; every other change (scalars, lists, type +changes) is a plain full-value assignment. Removed keys are always emitted +as ``$delete``, even when the object becomes empty. The output stays a +subset of jsondiff's marshaled syntax, so ``jsondiff.patch(..., +marshal=True)`` can apply it, and our patcher can apply +jsondiff-produced diffs (including positional ``$insert``/``$delete`` +list edits and per-index nested diffs). +""" from kanta.structs import ChangeRecord -from kanta.serialization.base import ReplayResult, replay +from kanta.serialization.base import ReplayResult, apply_diff, marshal_value, replay from kanta.serialization.framing import LineFramer from kanta.serialization.json import JsonSerializer +_UNCHANGED = object() + + +def _escape_key(key: str) -> str: + """Escape a user key for use as a diff key (``$foo`` -> ``$$foo``).""" + if isinstance(key, str) and key.startswith("$"): + return "$" + key + return key + + +def _diff(previous, current): + """Compute a raw diff, or _UNCHANGED if there is no difference.""" + if isinstance(previous, dict) and isinstance(current, dict): + result = {} + deleted = [_escape_key(k) for k in previous if k not in current] + if deleted: + result["$delete"] = deleted + for key, new_value in current.items(): + ekey = _escape_key(key) + if key not in previous: + result[ekey] = marshal_value(new_value) + else: + sub = _diff(previous[key], new_value) + if sub is not _UNCHANGED: + result[ekey] = sub + return result if result else _UNCHANGED + if previous == current: + return _UNCHANGED + if isinstance(current, dict): + # A dict assigned over a non-dict is ambiguous with a nested diff; + # like jsondiff, use $replace for that case. + return {"$replace": marshal_value(current)} + return marshal_value(current) + def compute_diff(previous: dict, current: dict) -> dict | None: - """Compute a jsondiff patch between two dicts. + """Compute a marshaled diff between two state dicts. Returns None if there is no difference. """ - return jsondiff.diff(previous, current, marshal=True) or None - - -def _apply_diff(state: dict, diff: dict) -> dict: - """Apply a jsondiff patch manually, handling ``$replace`` and ``$delete``. - - jsondiff.patch does not handle nested ``$replace`` commands when the - parent key is missing from the state. This function recursively applies - diffs, treating ``$replace`` as full replacement and ``$delete`` as - key removal. - """ - if not isinstance(diff, dict): - return diff - - result = dict(state) if isinstance(state, dict) else state - if not isinstance(result, dict): - result = {} - - for key, value in diff.items(): - if key == "$replace": - return value - elif key == "$delete": - if isinstance(value, list): - for k in value: - result.pop(k, None) - else: - result.pop(value, None) - elif isinstance(value, dict): - old = result.get(key, {}) - if not isinstance(old, dict): - old = {} - result[key] = _apply_diff(old, value) - else: - result[key] = value - - return result + diff = _diff(previous, current) + return diff if diff is not _UNCHANGED else None def patch_state(state: dict, diff: dict) -> dict: - """Apply a jsondiff patch to a state dict. - - The diff was produced with ``marshal=True`` (string keys like - ``"$replace"`` and ``"$delete"``) and decoded from JSON. - """ - return _apply_diff(state, diff) + """Apply a marshaled diff to a state dict.""" + return apply_diff(state, diff) # Backward-compatible JSONL replay using the default serializer. diff --git a/kanta/logging.py b/kanta/logging.py index b34fe44..2d9fff2 100644 --- a/kanta/logging.py +++ b/kanta/logging.py @@ -15,6 +15,7 @@ from typing import Any import msgspec +from kanta.serialization.base import _apply, unmarshal from kanta.tty import Line, displaywidth transaction_logger = logging.getLogger("kanta.transaction") @@ -312,6 +313,13 @@ def _collect_changes( changes.append(("update" if existed else "add", path, diff)) return + old_at_path = _get_nested(previous, path) + if isinstance(old_at_path, list): + # List edits ($insert/$delete/per-index) are shown as one whole-list + # update; the diff is already unmarshaled at this point. + changes.append(("update", path, _apply(old_at_path, diff))) + return + for key, value in diff.items(): if key == "$delete": if isinstance(value, list): @@ -340,7 +348,14 @@ def _collect_changes( ("update" if old_collection is not None else "add", path, value) ) elif isinstance(key, str) and key.startswith("$"): - changes.append(("add", path, {key: value})) + # Unknown $-command or (post-unmarshal) a user key starting with + # "$": treat as a normal key. + new_path = path + [str(key)] + existed = _get_nested(previous, new_path) is not None + if existed: + _collect_changes(value, new_path, changes, previous) + else: + changes.append(("add", new_path, value)) else: new_path = path + [str(key)] existed = _get_nested(previous, new_path) is not None @@ -418,7 +433,7 @@ def format_diff( Returns a list of formatted lines (without newlines). """ changes: list[tuple[str, list[str], Any]] = [] - _collect_changes(diff, [], changes, previous) + _collect_changes(unmarshal(diff), [], changes, previous) if not changes: return [] lines = [] diff --git a/kanta/serialization/base.py b/kanta/serialization/base.py index 9008c3e..abb37f3 100644 --- a/kanta/serialization/base.py +++ b/kanta/serialization/base.py @@ -121,33 +121,97 @@ def replay( def _patch_state(state: dict, diff: dict) -> dict: - return _apply_diff(state, diff) + return apply_diff(state, diff) -def _apply_diff(state: dict, diff: dict) -> dict: +_COMMAND_KEYS = frozenset({"$delete", "$insert", "$replace"}) + + +def _escape(value: str) -> str: + """Escape a user string for the marshaled format (``$x`` -> ``$$x``).""" + return "$" + value if value.startswith("$") else value + + +def _unescape(value: str) -> str: + """Reverse escaping; command strings pass through unchanged.""" + if value in _COMMAND_KEYS: + return value + if value.startswith("$"): + return value[1:] + return value + + +def marshal_value(value: Any) -> Any: + """Escape all ``$``-prefixed strings in a value stored in a diff.""" + if isinstance(value, dict): + return { + _escape(k) if isinstance(k, str) else k: marshal_value(v) + for k, v in value.items() + } + if isinstance(value, list): + return [marshal_value(v) for v in value] + if isinstance(value, str): + return _escape(value) + return value + + +def unmarshal(diff: Any) -> Any: + """Unescape a marshaled diff (keys, values and ``$delete`` entries).""" + if isinstance(diff, dict): + return { + _unescape(k) if isinstance(k, str) else k: unmarshal(v) + for k, v in diff.items() + } + if isinstance(diff, list): + return [unmarshal(v) for v in diff] + if isinstance(diff, str): + return _unescape(diff) + return diff + + +def apply_diff(state: Any, diff: Any) -> Any: + """Apply a marshaled jsondiff-format diff. + + Mirrors ``jsondiff.patch(..., marshal=True)``: supports ``$replace``, + key-based ``$delete`` on dicts, and positional ``$delete``/``$insert`` + plus per-index nested diffs on lists. + """ + return _apply(state, unmarshal(diff)) + + +def _apply(state: Any, diff: Any) -> Any: if not isinstance(diff, dict): return diff + if not diff: + return state + if "$replace" in diff: + return diff["$replace"] - result = dict(state) if isinstance(state, dict) else state - if not isinstance(result, dict): - result = {} + if isinstance(state, list): + result = list(state) + deletes = diff.get("$delete") + if deletes: + for pos in deletes: + result.pop(pos) + for pos, value in diff.get("$insert", []): + result.insert(pos, value) + for key, value in diff.items(): + if key in ("$delete", "$insert"): + continue + pos = int(key) + result[pos] = _apply(result[pos], value) + return result + result = dict(state) if isinstance(state, dict) else {} for key, value in diff.items(): - if key == "$replace": - return value if key == "$delete": - if isinstance(value, list): - for k in value: - result.pop(k, None) - else: - result.pop(value, None) + keys = value if isinstance(value, list) else [value] + for k in keys: + result.pop(k, None) + elif key == "$insert": continue - if isinstance(value, dict): - old = result.get(key, {}) - if not isinstance(old, dict): - old = {} - result[key] = _apply_diff(old, value) - continue - result[key] = value - + elif key in result: + result[key] = _apply(result[key], value) + else: + result[key] = value return result diff --git a/pyproject.toml b/pyproject.toml index 7896564..ac084ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ readme = "README.md" requires-python = ">=3.11" dependencies = [ "blake3>=1.0.8", - "jsondiff>=2.2.1", "msgspec>=0.20.0", ] @@ -35,6 +34,7 @@ Repository = "https://git.zi.fi/LeoVasanko/kanta" [dependency-groups] dev = [ + "jsondiff>=2.2.1", "pytest>=9.0.2", "pytest-asyncio>=1.3.0", ] diff --git a/tests/test_diff.py b/tests/test_diff.py index 67de6e6..f6fe410 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -1,8 +1,26 @@ -from kanta.diff import compute_diff +"""Tests for our own diff producer/consumer and jsondiff compatibility. + +jsondiff is a dev dependency used only here, to verify that: + +- jsondiff.patch(..., marshal=True) can apply patches produced by + compute_diff (our format is a subset of jsondiff's marshaled syntax); +- apply_diff can apply patches produced by jsondiff.diff(..., marshal=True), + including positional $insert/$delete list edits and per-index nested diffs. +""" + +import jsondiff +import pytest + +from kanta.diff import compute_diff, patch_state +from kanta.logging import format_diff +from kanta.serialization.base import apply_diff + +# --- Producer: compute_diff ------------------------------------------------ def test_no_diff(): assert compute_diff({"a": 1}, {"a": 1}) is None + assert compute_diff({}, {}) is None def test_simple_diff(): @@ -14,3 +32,199 @@ def test_simple_diff(): def test_nested_diff(): diff = compute_diff({"x": {"y": 1}}, {"x": {"y": 2}}) assert diff == {"x": {"y": 2}} + + +def test_key_added(): + assert compute_diff({"a": 1}, {"a": 1, "b": 2}) == {"b": 2} + + +def test_key_removed(): + assert compute_diff({"a": 1, "b": 2}, {"a": 1}) == {"$delete": ["b"]} + + +def test_last_key_removed_is_delete_not_replace(): + # jsondiff's minimal-diff search emits {"$replace": {}} here; we emit + # what actually happened: the key was deleted. + assert compute_diff({"a": 1}, {}) == {"$delete": ["a"]} + assert compute_diff({"x": {"y": 1}}, {"x": {}}) == {"x": {"$delete": ["y"]}} + + +def test_list_changes_are_full_assignment(): + # No $insert/$delete positional edits: lists are replaced wholesale. + assert compute_diff({"l": [1, 2]}, {"l": [1, 2, 3]}) == {"l": [1, 2, 3]} + assert compute_diff({"l": [1, 2, 3]}, {"l": [1, 3]}) == {"l": [1, 3]} + assert compute_diff({"l": [1]}, {"l": []}) == {"l": []} + + +def test_list_with_unchanged_prefix_is_full_assignment(): + diff = compute_diff({"l": ["a", "b", "c"]}, {"l": ["a", "x", "b", "c"]}) + assert diff == {"l": ["a", "x", "b", "c"]} + + +def test_type_changes_are_full_assignment(): + assert compute_diff({"a": {"x": 1}}, {"a": [1]}) == {"a": [1]} + # A dict replacing a non-dict needs $replace (a bare dict would read as + # a nested diff); this matches jsondiff. + assert compute_diff({"a": [1]}, {"a": {"x": 1}}) == {"a": {"$replace": {"x": 1}}} + assert compute_diff({"a": 1}, {"a": None}) == {"a": None} + + +def test_new_dict_value_assigned_wholesale(): + assert compute_diff({}, {"a": {"x": 1}}) == {"a": {"x": 1}} + + +def test_dollar_keys_escaped(): + assert compute_diff({}, {"$weird": 1}) == {"$$weird": 1} + assert compute_diff({"$weird": 1}, {"$weird": 2}) == {"$$weird": 2} + assert compute_diff({"$weird": 1}, {}) == {"$delete": ["$$weird"]} + + +def test_dollar_values_escaped(): + assert compute_diff({"s": 1}, {"s": "$y"}) == {"s": "$$y"} + assert compute_diff({"s": 1}, {"s": "$delete"}) == {"s": "$$delete"} + # Nested values in wholesale assignments are escaped too. + assert compute_diff({}, {"o": {"s": "$y", "l": ["$z"]}}) == { + "o": {"s": "$$y", "l": ["$$z"]} + } + + +# --- Consumer: apply_diff / patch_state ------------------------------------- + + +def test_patch_state_delegates(): + assert patch_state({"a": 1}, {"a": 2}) == {"a": 2} + + +def test_apply_scalar_and_add(): + assert apply_diff({"a": 1}, {"a": 2, "b": 3}) == {"a": 2, "b": 3} + + +def test_apply_delete(): + assert apply_diff({"a": 1, "b": 2}, {"$delete": ["b"]}) == {"a": 1} + assert apply_diff({"a": 1}, {"$delete": ["a"]}) == {} + + +def test_apply_replace(): + assert apply_diff({"a": 1, "b": 2}, {"$replace": {"c": 3}}) == {"c": 3} + assert apply_diff({"x": {"a": 1}}, {"x": {"$replace": [1]}}) == {"x": [1]} + + +def test_apply_nested_delete(): + diff = {"x": {"$delete": ["y"]}} + assert apply_diff({"x": {"y": 2, "z": 3}}, diff) == {"x": {"z": 3}} + + +def test_apply_list_insert(): + diff = {"l": {"$insert": [[1, "x"]]}} + assert apply_diff({"l": ["a", "b"]}, diff) == {"l": ["a", "x", "b"]} + + +def test_apply_list_delete(): + diff = {"l": {"$delete": [1]}} + assert apply_diff({"l": ["a", "b", "c"]}, diff) == {"l": ["a", "c"]} + + +def test_apply_list_delete_multiple_positions(): + # jsondiff emits positions in descending order for sequential pops. + diff = {"l": {"$delete": [4, 2, 0]}} + assert apply_diff({"l": [0, 1, 2, 3, 4]}, diff) == {"l": [1, 3]} + + +def test_apply_list_insert_and_delete(): + diff = {"l": {"$insert": [[0, 9], [2, 8], [4, 7]], "$delete": [2, 0]}} + assert apply_diff({"l": [0, 1, 2, 3]}, diff) == {"l": [9, 1, 8, 3, 7]} + + +def test_apply_list_per_index_nested_diff(): + diff = {"l": {"1": {"y": 3}}} + state = {"l": [{"x": 1}, {"y": 2}]} + assert apply_diff(state, diff) == {"l": [{"x": 1}, {"y": 3}]} + + +def test_apply_escaped_keys_and_values(): + assert apply_diff({}, {"$$weird": 1}) == {"$weird": 1} + assert apply_diff({"$weird": 1}, {"$delete": ["$$weird"]}) == {} + assert apply_diff({"s": 1}, {"s": "$$y"}) == {"s": "$y"} + assert apply_diff({"s": 1}, {"s": "$$delete"}) == {"s": "$delete"} + assert apply_diff({}, {"o": {"s": "$$y", "l": ["$$z"]}}) == { + "o": {"s": "$y", "l": ["$z"]} + } + + +def test_apply_empty_diff(): + assert apply_diff({"a": 1}, {}) == {"a": 1} + + +def test_apply_diff_on_missing_state(): + assert apply_diff({}, {"a": {"b": 1}}) == {"a": {"b": 1}} + + +# --- jsondiff compatibility, both directions -------------------------------- + +COMPAT_CASES = [ + ("scalar change", {"a": 1}, {"a": 2}), + ("key add", {"a": 1}, {"a": 1, "b": 2}), + ("key remove", {"a": 1, "b": 2}, {"a": 1}), + ("last key removed", {"a": 1}, {}), + ("nested delete", {"a": {"x": 1, "y": 2}}, {"a": {"x": 1}}), + ("nested mixed", {"a": {"x": 1, "y": 2}}, {"a": {"x": 9, "z": 3}}), + ("list append", {"l": [1, 2]}, {"l": [1, 2, 3]}), + ("list insert mid", {"l": [1, 2, 3]}, {"l": [1, 9, 2, 3]}), + ("list remove mid", {"l": [1, 2, 3]}, {"l": [1, 3]}), + ("list remove many", {"l": [0, 1, 2, 3, 4]}, {"l": [1, 3]}), + ("list replace all", {"l": [1, 2]}, {"l": [3, 4]}), + ("list insert+delete", {"l": [0, 1, 2, 3]}, {"l": [9, 1, 8, 3, 7]}), + ("dict in list", {"l": [{"x": 1}, {"y": 2}]}, {"l": [{"x": 1}, {"y": 3}]}), + ("list to empty", {"l": [1]}, {"l": []}), + ("type change dict->list", {"a": {"x": 1}}, {"a": [1]}), + ("type change list->dict", {"a": [1]}, {"a": {"x": 1}}), + ("dollar key", {"$k": 1, "b": 1}, {"$k": 2}), + ("dollar value", {"s": "$x"}, {"s": "$y"}), + ( + "deep nesting", + {"a": {"b": {"c": {"d": 1, "e": 2}}}}, + {"a": {"b": {"c": {"d": 9}}}}, + ), +] + + +@pytest.mark.parametrize("name,old,new", COMPAT_CASES, ids=[c[0] for c in COMPAT_CASES]) +def test_jsondiff_applies_our_patches(name, old, new): + diff = compute_diff(old, new) + assert diff is not None + assert jsondiff.patch(old, diff, marshal=True) == new + + +@pytest.mark.parametrize("name,old,new", COMPAT_CASES, ids=[c[0] for c in COMPAT_CASES]) +def test_we_apply_jsondiff_patches(name, old, new): + diff = jsondiff.diff(old, new, marshal=True) + assert apply_diff(old, diff) == new + + +@pytest.mark.parametrize("name,old,new", COMPAT_CASES, ids=[c[0] for c in COMPAT_CASES]) +def test_our_own_round_trip(name, old, new): + diff = compute_diff(old, new) + assert diff is not None + assert apply_diff(old, diff) == new + + +def test_no_diff_means_equal_states(): + for _name, old, new in COMPAT_CASES: + assert compute_diff(old, new) is not None # cases really differ + assert compute_diff({"a": [1, {"b": "$x"}]}, {"a": [1, {"b": "$x"}]}) is None + + +# --- Logging ---------------------------------------------------------------- + + +def test_format_diff_list_edit_shows_whole_list(): + diff = jsondiff.diff({"l": [1, 2, 3]}, {"l": [1, 9, 3]}, marshal=True) + lines = format_diff(diff, previous={"l": [1, 2, 3]}) + text = "\n".join(lines) + assert "$insert" not in text + assert "[1, 9, 3]" in text + + +def test_format_diff_unescapes_dollar_keys(): + lines = format_diff({"$$weird": 1}, previous={}) + assert any("$weird" in line and "$$weird" not in line for line in lines) diff --git a/tests/test_kanta_integration.py b/tests/test_kanta_integration.py index 7971575..563ec39 100644 --- a/tests/test_kanta_integration.py +++ b/tests/test_kanta_integration.py @@ -47,7 +47,7 @@ async def test_new_file_writes_bootstrap_record_without_handlers( records = read_changes(path, format_config) assert len(records) == 1 assert records[0].a == "bootstrap" - assert records[0].diff == {"$replace": {"users": {}, "counter": 0}} + assert records[0].diff == {"users": {}, "counter": 0} @pytest.mark.asyncio @@ -63,7 +63,8 @@ async def test_new_file_persists_initial_state_for_roundtrip(tmp_path, format_co assert len(records) == 1 assert records[0].a == "bootstrap" assert records[0].diff == { - "$replace": {"users": {"alice": {"name": "Alice", "age": 0}}, "counter": 5} + "users": {"alice": {"name": "Alice", "age": 0}}, + "counter": 5, } kanta2 = make_kanta(path, Data, format_config) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 8cd8452..f881063 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -183,7 +183,7 @@ def test_apply_returns_change_information(): assert result.migrations[0].name == "migrate_v1" assert result.migrations[0].description == "Set x" assert result.migrations[0].changed is True - assert result.migrations[0].diff == {"$replace": {"x": 1}} + assert result.migrations[0].diff == {"x": 1} assert result.migrations[1].name == "migrate_v2" assert result.migrations[1].description == "No-op"