Fixes to JSONL management, starting from empty state rather than default DB.
This commit is contained in:
+23
-20
@@ -32,7 +32,7 @@ class _ChangeRecord(msgspec.Struct):
|
|||||||
_change_encoder = msgspec.json.Encoder()
|
_change_encoder = msgspec.json.Encoder()
|
||||||
|
|
||||||
|
|
||||||
async def load_jsonl(db_path: Path, empty_data: dict) -> dict:
|
async def load_jsonl(db_path: Path) -> dict:
|
||||||
"""Load data from disk by applying change log.
|
"""Load data from disk by applying change log.
|
||||||
|
|
||||||
Replays all changes from JSONL file using plain dicts (to handle
|
Replays all changes from JSONL file using plain dicts (to handle
|
||||||
@@ -40,29 +40,32 @@ async def load_jsonl(db_path: Path, empty_data: dict) -> dict:
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
db_path: Path to the JSONL database file
|
db_path: Path to the JSONL database file
|
||||||
empty_data: Empty data structure to start with (as dict)
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The final state after applying all changes
|
The final state after applying all changes
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If file doesn't exist or cannot be loaded
|
||||||
"""
|
"""
|
||||||
data_dict = empty_data.copy()
|
if not db_path.exists():
|
||||||
if db_path.exists():
|
raise ValueError(f"Database file not found: {db_path}")
|
||||||
try:
|
data_dict: dict = {}
|
||||||
# Read entire file at once and split into lines
|
try:
|
||||||
async with aiofiles.open(db_path, "rb") as f:
|
# Read entire file at once and split into lines
|
||||||
content = await f.read()
|
async with aiofiles.open(db_path, "rb") as f:
|
||||||
for line_num, line in enumerate(content.split(b"\n"), 1):
|
content = await f.read()
|
||||||
line = line.strip()
|
for line_num, line in enumerate(content.split(b"\n"), 1):
|
||||||
if not line:
|
line = line.strip()
|
||||||
continue
|
if not line:
|
||||||
try:
|
continue
|
||||||
change = msgspec.json.decode(line)
|
try:
|
||||||
# Apply the diff to current state (marshal=True for $-prefixed keys)
|
change = msgspec.json.decode(line)
|
||||||
data_dict = jsondiff.patch(data_dict, change["diff"], marshal=True)
|
# Apply the diff to current state (marshal=True for $-prefixed keys)
|
||||||
except Exception as e:
|
data_dict = jsondiff.patch(data_dict, change["diff"], marshal=True)
|
||||||
raise ValueError(f"Error parsing line {line_num}: {e}")
|
except Exception as e:
|
||||||
except (OSError, ValueError, msgspec.DecodeError) as e:
|
raise ValueError(f"Error parsing line {line_num}: {e}")
|
||||||
raise ValueError(f"Failed to load database: {e}")
|
except (OSError, ValueError, msgspec.DecodeError) as e:
|
||||||
|
raise ValueError(f"Failed to load database: {e}")
|
||||||
return data_dict
|
return data_dict
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+14
-5
@@ -73,13 +73,22 @@ class DB:
|
|||||||
self._current_actor: str = "system"
|
self._current_actor: str = "system"
|
||||||
|
|
||||||
async def load(self, db_path: str | None = None) -> None:
|
async def load(self, db_path: str | None = None) -> None:
|
||||||
"""Load data from JSONL change log."""
|
"""Load data from JSONL change log.
|
||||||
|
|
||||||
|
If file doesn't exist, keeps the initialized empty structure and
|
||||||
|
sets _previous_builtins to {} for creating a new database.
|
||||||
|
"""
|
||||||
if db_path is not None:
|
if db_path is not None:
|
||||||
self.db_path = Path(db_path)
|
self.db_path = Path(db_path)
|
||||||
empty = msgspec.to_builtins(self._data)
|
try:
|
||||||
data_dict = await load_jsonl(self.db_path, empty)
|
data_dict = await load_jsonl(self.db_path)
|
||||||
self._data = _json_decoder.decode(_json_encoder.encode(data_dict))
|
self._data = _json_decoder.decode(_json_encoder.encode(data_dict))
|
||||||
self._previous_builtins = msgspec.to_builtins(self._data)
|
# Track the JSONL file state directly - this is what we diff against
|
||||||
|
self._previous_builtins = data_dict
|
||||||
|
except ValueError:
|
||||||
|
if self.db_path.exists():
|
||||||
|
raise # File exists but failed to load - re-raise
|
||||||
|
# File doesn't exist: keep initialized _data, _previous_builtins stays {}
|
||||||
|
|
||||||
def _queue_change(self) -> None:
|
def _queue_change(self) -> None:
|
||||||
current = msgspec.to_builtins(self._data)
|
current = msgspec.to_builtins(self._data)
|
||||||
|
|||||||
Reference in New Issue
Block a user