From 943bda806dbbf4aac87c999121ad0551b40bef26 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Fri, 23 Jan 2026 19:56:44 +0000 Subject: [PATCH] Update migrate script with the latest database changes. --- paskia/migrate/__init__.py | 297 ++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 152 deletions(-) diff --git a/paskia/migrate/__init__.py b/paskia/migrate/__init__.py index 21bdb14..f45030e 100644 --- a/paskia/migrate/__init__.py +++ b/paskia/migrate/__init__.py @@ -13,8 +13,7 @@ Or via the CLI entry point (if installed): import asyncio from datetime import datetime, timezone - -import base64url +from uuid import UUID from paskia.authsession import EXPIRES @@ -36,13 +35,6 @@ SQL_DB_DEFAULT = "sqlite+aiosqlite:///paskia.sqlite" JSON_DB_DEFAULT = "paskia.jsonl" -def _bytes_to_str(b: bytes | None) -> str | None: - """Convert bytes to base64url string.""" - if b is None: - return None - return base64url.enc(b) - - async def migrate_from_sql( sql_db_path: str = SQL_DB_DEFAULT, json_db_path: str = JSON_DB_DEFAULT, @@ -76,167 +68,168 @@ async def migrate_from_sql( # Initialize destination JSON database json_db = JSONDB(json_db_path) - json_db.load() + await json_db.load() print(f"Migrating from {sql_db_path} to {json_db_path}...") # Build all data directly without saving (we'll save once at the end) - with json_db._lock: - # Track old permission ID -> new scope mapping for migration - # Also track org-specific admin permissions to consolidate - old_org_admin_pattern = re.compile(r"^auth:org:([0-9a-f-]+)$", re.IGNORECASE) - org_admin_uuids = set() # org UUIDs that had org-specific admin permissions + # Track old permission ID -> new scope mapping for migration + # Also track org-specific admin permissions to consolidate + old_org_admin_pattern = re.compile(r"^auth:org:([0-9a-f-]+)$", re.IGNORECASE) + org_admin_uuids = set() # org UUIDs that had org-specific admin permissions - # First pass: identify org-specific admin permissions - permissions = await sql_db.list_permissions() - for perm in permissions: - match = old_org_admin_pattern.match(perm.id) - if match: - org_admin_uuids.add(match.group(1).lower()) + # First pass: identify org-specific admin permissions + permissions = await sql_db.list_permissions() + for perm in permissions: + match = old_org_admin_pattern.match(perm.id) + if match: + org_admin_uuids.add(match.group(1).lower()) - # Migrate permissions with UUID keys and scope field - # Always create exactly one common auth:org:admin permission for all org admin needs - org_admin_perm_uuid = str(uuid7.create()) - json_db._data.permissions[org_admin_perm_uuid] = _PermissionData( - scope="auth:org:admin", - display_name="Org Admin", + # Migrate permissions with UUID keys and scope field + # Always create exactly one common auth:org:admin permission for all org admin needs + org_admin_perm_uuid: UUID = uuid7.create() + json_db._data.permissions[org_admin_perm_uuid] = _PermissionData( + scope="auth:org:admin", + display_name="Org Admin", + orgs={}, + ) + + # Mapping from old permission ID to new scope + perm_id_to_scope: dict[str, str] = {} + + for perm in permissions: + # Skip old org-specific admin permissions (auth:org:{uuid}) - they map to auth:org:admin + match = old_org_admin_pattern.match(perm.id) + if match: + perm_id_to_scope[perm.id] = "auth:org:admin" + continue + + # Skip if this is already auth:org:admin - we created one above + if perm.id == "auth:org:admin": + perm_id_to_scope[perm.id] = "auth:org:admin" + continue + + # Regular permission - create with UUID key + perm_uuid: UUID = uuid7.create() + json_db._data.permissions[perm_uuid] = _PermissionData( + scope=perm.id, # Old ID becomes the scope + display_name=perm.display_name, orgs={}, ) + perm_id_to_scope[perm.id] = perm.id # Scope same as old ID + print( + f" Migrated {len(permissions)} permissions (with {len(org_admin_uuids)} org-specific admins consolidated to auth:org:admin)" + ) - # Mapping from old permission ID to new scope - perm_id_to_scope: dict[str, str] = {} - - for perm in permissions: - # Skip old org-specific admin permissions (auth:org:{uuid}) - they map to auth:org:admin - match = old_org_admin_pattern.match(perm.id) - if match: - perm_id_to_scope[perm.id] = "auth:org:admin" - continue - - # Skip if this is already auth:org:admin - we created one above - if perm.id == "auth:org:admin": - perm_id_to_scope[perm.id] = "auth:org:admin" - continue - - # Regular permission - create with UUID key - perm_uuid = str(uuid7.create()) - json_db._data.permissions[perm_uuid] = _PermissionData( - scope=perm.id, # Old ID becomes the scope - display_name=perm.display_name, - orgs={}, - ) - perm_id_to_scope[perm.id] = perm.id # Scope same as old ID - print( - f" Migrated {len(permissions)} permissions (with {len(org_admin_uuids)} org-specific admins consolidated to auth:org:admin)" + # Migrate organizations + orgs = await sql_db.list_organizations() + for org in orgs: + org_key: UUID = org.uuid + json_db._data.orgs[org_key] = _OrgData( + display_name=org.display_name, ) + # Update permissions to allow this org to grant them (by scope) + for old_perm_id in org.permissions: + new_scope = perm_id_to_scope.get(old_perm_id, old_perm_id) + # Find permission with this scope and add org + for pid, p in json_db._data.permissions.items(): + if p.scope == new_scope: + p.orgs[org_key] = True + break + # Ensure every org can grant auth:org:admin + json_db._data.permissions[org_admin_perm_uuid].orgs[org_key] = True + print(f" Migrated {len(orgs)} organizations") - # Migrate organizations - orgs = await sql_db.list_organizations() - for org in orgs: - key = str(org.uuid) - json_db._data.orgs[key] = _OrgData( - display_name=org.display_name, - ) - # Update permissions to allow this org to grant them (by scope) - for old_perm_id in org.permissions: + # Migrate roles - convert old permission IDs to scopes + role_count = 0 + for org in orgs: + for role in org.roles: + role_key: UUID = role.uuid + # Convert old permission IDs to scopes + new_permissions = {} + for old_perm_id in role.permissions or []: new_scope = perm_id_to_scope.get(old_perm_id, old_perm_id) - # Find permission with this scope and add org - for pid, p in json_db._data.permissions.items(): - if p.scope == new_scope: - p.orgs[key] = True - break - # Ensure every org can grant auth:org:admin - json_db._data.permissions[org_admin_perm_uuid].orgs[key] = True - print(f" Migrated {len(orgs)} organizations") + new_permissions[new_scope] = True + json_db._data.roles[role_key] = _RoleData( + org=role.org_uuid, + display_name=role.display_name, + permissions=new_permissions, + ) + role_count += 1 + print(f" Migrated {role_count} roles") - # Migrate roles - convert old permission IDs to scopes - role_count = 0 - for org in orgs: - for role in org.roles: - key = str(role.uuid) - # Convert old permission IDs to scopes - new_permissions = {} - for old_perm_id in role.permissions or []: - new_scope = perm_id_to_scope.get(old_perm_id, old_perm_id) - new_permissions[new_scope] = True - json_db._data.roles[key] = _RoleData( - org=str(role.org_uuid), - display_name=role.display_name, - permissions=new_permissions, - ) - role_count += 1 - print(f" Migrated {role_count} roles") + # Migrate users + async with sql_db.session() as session: + result = await session.execute(select(UserModel)) + user_models = result.scalars().all() + for um in user_models: + user = um.as_dataclass() + user_key: UUID = user.uuid + json_db._data.users[user_key] = _UserData( + display_name=user.display_name, + role=user.role_uuid, + created_at=user.created_at or datetime.now(timezone.utc), + last_seen=user.last_seen, + visits=user.visits, + ) + print(f" Migrated {len(user_models)} users") - # Migrate users - async with sql_db.session() as session: - result = await session.execute(select(UserModel)) - user_models = result.scalars().all() - for um in user_models: - user = um.as_dataclass() - key = str(user.uuid) - json_db._data.users[key] = _UserData( - display_name=user.display_name, - role=str(user.role_uuid), - created_at=user.created_at or datetime.now(timezone.utc), - last_seen=user.last_seen, - visits=user.visits, - ) - print(f" Migrated {len(user_models)} users") + # Migrate credentials + async with sql_db.session() as session: + result = await session.execute(select(CredentialModel)) + cred_models = result.scalars().all() + for cm in cred_models: + cred = cm.as_dataclass() + cred_key: UUID = cred.uuid + json_db._data.credentials[cred_key] = _CredentialData( + credential_id=cred.credential_id, + user=cred.user_uuid, + aaguid=cred.aaguid, + public_key=cred.public_key, + sign_count=cred.sign_count, + created_at=cred.created_at, + last_used=cred.last_used, + last_verified=cred.last_verified, + ) + print(f" Migrated {len(cred_models)} credentials") - # Migrate credentials - async with sql_db.session() as session: - result = await session.execute(select(CredentialModel)) - cred_models = result.scalars().all() - for cm in cred_models: - cred = cm.as_dataclass() - key = str(cred.uuid) - json_db._data.credentials[key] = _CredentialData( - credential_id=cred.credential_id, - user=str(cred.user_uuid), - aaguid=str(cred.aaguid), - public_key=cred.public_key, - sign_count=cred.sign_count, - created_at=cred.created_at, - last_used=cred.last_used, - last_verified=cred.last_verified, - ) - print(f" Migrated {len(cred_models)} credentials") + # Migrate sessions + async with sql_db.session() as session: + result = await session.execute(select(SessionModel)) + session_models = result.scalars().all() + for sm in session_models: + sess = sm.as_dataclass() + session_key: bytes = sess.key + json_db._data.sessions[session_key] = _SessionData( + user=sess.user_uuid, + credential=sess.credential_uuid, + host=sess.host, + ip=sess.ip, + user_agent=sess.user_agent, + expiry=sess.renewed + EXPIRES, # Convert renewed to expiry + ) + print(f" Migrated {len(session_models)} sessions") - # Migrate sessions - async with sql_db.session() as session: - result = await session.execute(select(SessionModel)) - session_models = result.scalars().all() - for sm in session_models: - sess = sm.as_dataclass() - key_b64 = _bytes_to_str(sess.key) - json_db._data.sessions[key_b64] = _SessionData( - user=str(sess.user_uuid), - credential=str(sess.credential_uuid), - host=sess.host, - ip=sess.ip, - user_agent=sess.user_agent, - expiry=sess.renewed + EXPIRES, # Convert renewed to expiry - ) - print(f" Migrated {len(session_models)} sessions") + # Migrate reset tokens + async with sql_db.session() as session: + result = await session.execute(select(ResetTokenModel)) + token_models = result.scalars().all() + for tm in token_models: + token = tm.as_dataclass() + token_key: bytes = token.key + json_db._data.reset_tokens[token_key] = _ResetTokenData( + user=token.user_uuid, + expiry=token.expiry, + token_type=token.token_type, + ) + print(f" Migrated {len(token_models)} reset tokens") - # Migrate reset tokens - async with sql_db.session() as session: - result = await session.execute(select(ResetTokenModel)) - token_models = result.scalars().all() - for tm in token_models: - token = tm.as_dataclass() - key_b64 = _bytes_to_str(token.key) - json_db._data.reset_tokens[key_b64] = _ResetTokenData( - user=str(token.user_uuid), - expiry=token.expiry, - token_type=token.token_type, - ) - print(f" Migrated {len(token_models)} reset tokens") + # Queue and flush all changes with actor "migrate" + json_db._current_actor = "migrate" + json_db._queue_change() + from paskia.db.jsonl import flush_changes - # Queue and flush all changes with actor "migrate" - json_db._current_actor = "migrate" - json_db._queue_change() - json_db.flush() + await flush_changes(json_db.db_path, json_db._pending_changes) print("Migration complete!")