Remove paskia-migration script (SQL no longer supported).

This commit is contained in:
2026-02-05 19:32:18 +00:00
parent 250189dbe5
commit 5deb57435b
6 changed files with 6 additions and 711 deletions
+4 -4
View File
@@ -67,7 +67,7 @@ def create_change_record(
# Actions that are allowed to create a new database file # Actions that are allowed to create a new database file
_BOOTSTRAP_ACTIONS = frozenset({"bootstrap", "migrate:sql"}) _BOOTSTRAP_ACTIONS = frozenset({"bootstrap"})
async def flush_changes( async def flush_changes(
@@ -91,7 +91,7 @@ async def flush_changes(
if first_action not in _BOOTSTRAP_ACTIONS: if first_action not in _BOOTSTRAP_ACTIONS:
_logger.error( _logger.error(
"Refusing to create database file with action '%s' - " "Refusing to create database file with action '%s' - "
"only bootstrap or migrate can create a new database", "only bootstrap can create a new database",
first_action, first_action,
) )
pending_changes.clear() pending_changes.clear()
@@ -233,8 +233,8 @@ class JsonlStore:
# Check for out-of-transaction modifications # Check for out-of-transaction modifications
current_state = msgspec.to_builtins(self.db) current_state = msgspec.to_builtins(self.db)
if current_state != self._previous_builtins: if current_state != self._previous_builtins:
# Allow bootstrap/migrate to create a new database from empty state # Allow bootstrap to create a new database from empty state
is_bootstrap = action in _BOOTSTRAP_ACTIONS or action.startswith("migrate:") is_bootstrap = action in _BOOTSTRAP_ACTIONS
if is_bootstrap and not self._previous_builtins: if is_bootstrap and not self._previous_builtins:
pass # Expected: creating database from scratch pass # Expected: creating database from scratch
else: else:
+1 -1
View File
@@ -19,7 +19,7 @@ migrations = sorted(
key=lambda f: int(f.__name__.removeprefix("migrate_v")), key=lambda f: int(f.__name__.removeprefix("migrate_v")),
) )
DBVER = len(migrations) # Used by bootstrap and migrate:sql to set initial version DBVER = len(migrations) # Used by bootstrap to set initial version
async def apply_all_migrations( async def apply_all_migrations(
+1 -1
View File
@@ -734,7 +734,7 @@ def bootstrap(
- Admin user with Administration role - Admin user with Administration role
- Reset token for admin registration - Reset token for admin registration
This is the only way to create a new database file (besides migrate). This is the only way to create a new database file.
All data is created atomically - if any step fails, nothing is written. All data is created atomically - if any step fails, nothing is written.
Args: Args:
-262
View File
@@ -1,262 +0,0 @@
"""
SQL to JSON migration module for Paskia.
This module contains the legacy SQL database implementation and migration tools
for converting from the old SQLite database to the new JSONL format.
Usage:
python -m paskia.migrate --sql sqlite+aiosqlite:///paskia.sqlite --json paskia.jsonl
Or via the CLI entry point (if installed):
paskia-migrate --sql sqlite+aiosqlite:///paskia.sqlite --json paskia.jsonl
"""
import argparse
import asyncio
import re
from datetime import UTC, datetime
from uuid import UUID
import base64url
import uuid7
from sqlalchemy import select
from paskia.authsession import EXPIRES
from paskia.db.jsonl import JsonlStore
from paskia.db.structs import (
DB,
Credential,
Org,
Permission,
Role,
Session,
User,
)
from .sql import (
DB as SQLDB,
)
from .sql import (
CredentialModel,
SessionModel,
UserModel,
)
# Re-export for convenience
__all__ = ["migrate_from_sql", "main", "SQLDB"]
# Default paths
SQL_DB_DEFAULT = "sqlite+aiosqlite:///paskia.sqlite"
JSON_DB_DEFAULT = "paskia.jsonl"
async def migrate_from_sql(
sql_db_path: str = SQL_DB_DEFAULT,
json_db_path: str = JSON_DB_DEFAULT,
) -> None:
"""Migrate data from SQL database to JSON format.
Args:
sql_db_path: SQLAlchemy connection string for the source SQL database
json_db_path: Path for the destination JSONL file
"""
# Initialize source SQL database
sql_db = SQLDB(sql_db_path)
await sql_db.init_db()
# Initialize destination JSON database (fresh, don't load existing)
db = DB()
store = JsonlStore(db, json_db_path)
db._store = store
print(f"Migrating from {sql_db_path} to {json_db_path}...")
# Build all data directly without saving (we'll save once at the end)
# Track old permission ID -> new scope mapping for migration
# Also track org-specific admin permissions to consolidate
old_org_admin_pattern = re.compile(r"^auth:org:([0-9a-f-]+)$", re.IGNORECASE)
org_admin_uuids = set() # org UUIDs that had org-specific admin permissions
# First pass: identify org-specific admin permissions
permissions = await sql_db.list_permissions()
for perm in permissions:
match = old_org_admin_pattern.match(perm.id)
if match:
org_admin_uuids.add(match.group(1).lower())
# Migrate permissions with UUID keys and scope field
# Always create exactly one common auth:org:admin permission for all org admin needs
org_admin_perm_uuid: UUID = uuid7.create()
org_admin_perm = Permission(
scope="auth:org:admin",
display_name="Org Admin",
orgs={},
)
org_admin_perm.uuid = org_admin_perm_uuid
db.permissions[org_admin_perm_uuid] = org_admin_perm
# Mapping from old permission ID to new permission UUID
perm_id_to_uuid: dict[str, UUID] = {}
for perm in permissions:
# Skip old org-specific admin permissions (auth:org:{uuid}) - they map to auth:org:admin
match = old_org_admin_pattern.match(perm.id)
if match:
perm_id_to_uuid[perm.id] = org_admin_perm_uuid
continue
# Skip if this is already auth:org:admin - we created one above
if perm.id == "auth:org:admin":
perm_id_to_uuid[perm.id] = org_admin_perm_uuid
continue
# Regular permission - create with UUID key
perm_uuid: UUID = uuid7.create()
new_perm = Permission(
scope=perm.id, # Old ID becomes the scope
display_name=perm.display_name,
orgs={},
)
new_perm.uuid = perm_uuid
db.permissions[perm_uuid] = new_perm
perm_id_to_uuid[perm.id] = perm_uuid
print(
f" Migrated {len(permissions)} permissions (with {len(org_admin_uuids)} org-specific admins consolidated to auth:org:admin)"
)
# Migrate organizations
orgs = await sql_db.list_organizations()
for org in orgs:
org_key: UUID = org.uuid
new_org = Org(display_name=org.display_name)
new_org.uuid = org_key
db.orgs[org_key] = new_org
# Update permissions to allow this org to grant them (by UUID)
for old_perm_id in org.permissions:
perm_uuid = perm_id_to_uuid.get(old_perm_id)
if perm_uuid and perm_uuid in db.permissions:
db.permissions[perm_uuid].orgs[org_key] = True
# Ensure every org can grant auth:org:admin
db.permissions[org_admin_perm_uuid].orgs[org_key] = True
print(f" Migrated {len(orgs)} organizations")
# Migrate roles - convert old permission IDs to UUIDs
role_count = 0
for org in orgs:
for role in org.roles:
role_key: UUID = role.uuid
# Convert old permission IDs to UUIDs
new_permissions: dict[UUID, bool] = {}
for old_perm_id in role.permissions or []:
perm_uuid = perm_id_to_uuid.get(old_perm_id)
if perm_uuid:
new_permissions[perm_uuid] = True
new_role = Role(
org_uuid=role.org_uuid,
display_name=role.display_name,
permissions=new_permissions,
)
new_role.uuid = role_key
db.roles[role_key] = new_role
role_count += 1
print(f" Migrated {role_count} roles")
# Migrate users
async with sql_db.session() as session:
result = await session.execute(select(UserModel))
user_models = result.scalars().all()
for um in user_models:
legacy_user = um.as_dataclass()
user_key: UUID = legacy_user.uuid
new_user = User(
display_name=legacy_user.display_name,
role_uuid=legacy_user.role_uuid,
created_at=legacy_user.created_at or datetime.now(UTC),
last_seen=legacy_user.last_seen,
visits=legacy_user.visits,
)
new_user.uuid = user_key
db.users[user_key] = new_user
print(f" Migrated {len(user_models)} users")
# Migrate credentials
async with sql_db.session() as session:
result = await session.execute(select(CredentialModel))
cred_models = result.scalars().all()
for cm in cred_models:
legacy_cred = cm.as_dataclass()
cred_key: UUID = legacy_cred.uuid
new_cred = Credential(
credential_id=legacy_cred.credential_id,
user_uuid=legacy_cred.user_uuid,
aaguid=legacy_cred.aaguid,
public_key=legacy_cred.public_key,
sign_count=legacy_cred.sign_count,
created_at=legacy_cred.created_at,
last_used=legacy_cred.last_used,
last_verified=legacy_cred.last_verified,
)
new_cred.uuid = cred_key
db.credentials[cred_key] = new_cred
print(f" Migrated {len(cred_models)} credentials")
# Migrate sessions
# Old format: b"sess" + 12 bytes -> New format: base64url string (16 chars)
async with sql_db.session() as session:
result = await session.execute(select(SessionModel))
session_models = result.scalars().all()
for sm in session_models:
sess = sm.as_dataclass()
old_key: bytes = sess.key
# Strip b"sess" prefix and encode remaining 12 bytes as base64url
if old_key.startswith(b"sess"):
session_key = base64url.enc(old_key[4:])
else:
# Already in new format or unknown - try to use as-is
session_key = base64url.enc(old_key[:12])
db.sessions[session_key] = Session(
user_uuid=sess.user_uuid,
credential_uuid=sess.credential_uuid,
host=sess.host,
ip=sess.ip,
user_agent=sess.user_agent,
expiry=sess.renewed + EXPIRES, # Convert renewed to expiry
)
print(f" Migrated {len(session_models)} sessions")
# Reset tokens are not migrated - they will expire naturally
# and users can generate new ones as needed
print(" Reset tokens dropped (not migrated)")
# Queue and flush all changes using the transaction mechanism
with db.transaction("migrate:sql"):
pass # All data already added to _data, transaction commits on exit
await store.flush()
print("Migration complete!")
def main():
"""CLI entry point for migration."""
parser = argparse.ArgumentParser(
description="Migrate Paskia database from SQL to JSON"
)
parser.add_argument(
"--sql",
default=SQL_DB_DEFAULT,
help=f"Source SQL database connection string (default: {SQL_DB_DEFAULT})",
)
parser.add_argument(
"--json",
default=JSON_DB_DEFAULT,
help=f"Destination JSONL file path (default: {JSON_DB_DEFAULT})",
)
args = parser.parse_args()
asyncio.run(migrate_from_sql(args.sql, args.json))
if __name__ == "__main__":
main()
-438
View File
@@ -1,438 +0,0 @@
"""
Legacy SQL database implementation for migration purposes.
This module provides the async SQLAlchemy database layer that was used
before the JSONL format. It is kept here for migration purposes only.
DO NOT use this module for new code. Use paskia.db instead.
"""
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import UUID
from sqlalchemy import (
DateTime,
ForeignKey,
Integer,
LargeBinary,
String,
event,
select,
)
from sqlalchemy.dialects.sqlite import BLOB
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
# Legacy User class for SQL schema (uses 'role_uuid' not 'role')
@dataclass
class _LegacyUser:
"""User as stored in the old SQL schema with role_uuid field."""
uuid: UUID
display_name: str
role_uuid: UUID
created_at: datetime | None = None
last_seen: datetime | None = None
visits: int = 0
# Legacy Credential class for SQL schema (uses 'user_uuid' not 'user')
@dataclass
class _LegacyCredential:
"""Credential as stored in the old SQL schema with user_uuid field."""
uuid: UUID
credential_id: bytes
user_uuid: UUID
aaguid: UUID
public_key: bytes
sign_count: int
created_at: datetime
last_used: datetime | None = None
last_verified: datetime | None = None
# Legacy Role class for SQL schema (uses 'org_uuid' not 'org')
@dataclass
class _LegacyRole:
"""Role as stored in the old SQL schema with org_uuid field."""
uuid: UUID
org_uuid: UUID
display_name: str
permissions: list[str] | None = None
# Legacy Org class for SQL schema (has mutable permissions/roles lists)
@dataclass
class _LegacyOrg:
"""Org as stored in the old SQL schema with mutable permissions/roles."""
uuid: UUID
display_name: str
permissions: list[str] | None = None
roles: list[_LegacyRole] | None = None
# Legacy Session class for SQL schema (uses 'key' as field, 'user_uuid', 'credential_uuid')
@dataclass
class _LegacySession:
"""Session as stored in the old SQL schema."""
key: bytes
user_uuid: UUID
credential_uuid: UUID
host: str
ip: str
user_agent: str
renewed: datetime
# Legacy ResetToken class for SQL schema (uses 'key' as field, 'user_uuid')
@dataclass
class _LegacyResetToken:
"""ResetToken as stored in the old SQL schema."""
key: bytes
user_uuid: UUID
token_type: str
expiry: datetime
# Local Permission class for SQL schema (uses 'id' not 'uuid' + 'scope')
@dataclass
class SqlPermission:
"""Permission as stored in the old SQL schema with id field."""
id: str
display_name: str
DB_PATH_DEFAULT = "sqlite+aiosqlite:///paskia.sqlite"
def _normalize_dt(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
class Base(DeclarativeBase):
pass
class OrgModel(Base):
__tablename__ = "orgs"
uuid: Mapped[bytes] = mapped_column(LargeBinary(16), primary_key=True)
display_name: Mapped[str] = mapped_column(String, nullable=False)
def as_dataclass(self):
# Base Org without permissions/roles (filled by data accessors)
return _LegacyOrg(
uuid=UUID(bytes=self.uuid),
display_name=self.display_name,
)
@staticmethod
def from_dataclass(org: _LegacyOrg):
return OrgModel(uuid=org.uuid.bytes, display_name=org.display_name)
class RoleModel(Base):
__tablename__ = "roles"
uuid: Mapped[bytes] = mapped_column(LargeBinary(16), primary_key=True)
org_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("orgs.uuid", ondelete="CASCADE"), nullable=False
)
display_name: Mapped[str] = mapped_column(String, nullable=False)
def as_dataclass(self):
# Base Role without permissions (filled by data accessors)
return _LegacyRole(
uuid=UUID(bytes=self.uuid),
org_uuid=UUID(bytes=self.org_uuid),
display_name=self.display_name,
)
@staticmethod
def from_dataclass(role: _LegacyRole):
return RoleModel(
uuid=role.uuid.bytes,
org_uuid=role.org_uuid.bytes,
display_name=role.display_name,
)
class UserModel(Base):
__tablename__ = "users"
uuid: Mapped[bytes] = mapped_column(LargeBinary(16), primary_key=True)
display_name: Mapped[str] = mapped_column(String, nullable=False)
role_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("roles.uuid", ondelete="CASCADE"), nullable=False
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(UTC)
)
last_seen: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
visits: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
def as_dataclass(self) -> "_LegacyUser":
return _LegacyUser(
uuid=UUID(bytes=self.uuid),
display_name=self.display_name,
role_uuid=UUID(bytes=self.role_uuid),
created_at=_normalize_dt(self.created_at) or self.created_at,
last_seen=_normalize_dt(self.last_seen) or self.last_seen,
visits=self.visits,
)
@staticmethod
def from_dataclass(user: "_LegacyUser"):
return UserModel(
uuid=user.uuid.bytes,
display_name=user.display_name,
role_uuid=user.role_uuid.bytes,
created_at=user.created_at or datetime.now(UTC),
last_seen=user.last_seen,
visits=user.visits,
)
class CredentialModel(Base):
__tablename__ = "credentials"
uuid: Mapped[bytes] = mapped_column(LargeBinary(16), primary_key=True)
credential_id: Mapped[bytes] = mapped_column(
LargeBinary(64), unique=True, index=True
)
user_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("users.uuid", ondelete="CASCADE")
)
aaguid: Mapped[bytes] = mapped_column(LargeBinary(16), nullable=False)
public_key: Mapped[bytes] = mapped_column(BLOB, nullable=False)
sign_count: Mapped[int] = mapped_column(Integer, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(UTC)
)
last_used: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
last_verified: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
def as_dataclass(self):
return _LegacyCredential(
uuid=UUID(bytes=self.uuid),
credential_id=self.credential_id,
user_uuid=UUID(bytes=self.user_uuid),
aaguid=UUID(bytes=self.aaguid),
public_key=self.public_key,
sign_count=self.sign_count,
created_at=_normalize_dt(self.created_at) or self.created_at,
last_used=_normalize_dt(self.last_used) or self.last_used,
last_verified=_normalize_dt(self.last_verified) or self.last_verified,
)
class SessionModel(Base):
__tablename__ = "sessions"
key: Mapped[bytes] = mapped_column(LargeBinary(16), primary_key=True)
user_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("users.uuid", ondelete="CASCADE"), nullable=False
)
credential_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16),
ForeignKey("credentials.uuid", ondelete="CASCADE"),
nullable=False,
)
host: Mapped[str] = mapped_column(String, nullable=False)
ip: Mapped[str] = mapped_column(String(64), nullable=False)
user_agent: Mapped[str] = mapped_column(String(512), nullable=False)
renewed: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(UTC),
nullable=False,
)
def as_dataclass(self):
return _LegacySession(
key=self.key,
user_uuid=UUID(bytes=self.user_uuid),
credential_uuid=UUID(bytes=self.credential_uuid),
host=self.host,
ip=self.ip,
user_agent=self.user_agent,
renewed=_normalize_dt(self.renewed) or self.renewed,
)
@staticmethod
def from_dataclass(session: _LegacySession):
return SessionModel(
key=session.key,
user_uuid=session.user_uuid.bytes,
credential_uuid=session.credential_uuid.bytes,
host=session.host,
ip=session.ip,
user_agent=session.user_agent,
renewed=session.renewed,
)
class ResetTokenModel(Base):
__tablename__ = "reset_tokens"
key: Mapped[bytes] = mapped_column(LargeBinary(16), primary_key=True)
user_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("users.uuid", ondelete="CASCADE"), nullable=False
)
token_type: Mapped[str] = mapped_column(String, nullable=False)
expiry: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
def as_dataclass(self) -> _LegacyResetToken:
return _LegacyResetToken(
key=self.key,
user_uuid=UUID(bytes=self.user_uuid),
token_type=self.token_type,
expiry=_normalize_dt(self.expiry) or self.expiry,
)
class PermissionModel(Base):
__tablename__ = "permissions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
display_name: Mapped[str] = mapped_column(String, nullable=False)
def as_dataclass(self):
return SqlPermission(self.id, self.display_name)
@staticmethod
def from_dataclass(permission: SqlPermission):
return PermissionModel(
id=permission.id,
display_name=permission.display_name,
)
class OrgPermission(Base):
"""Permissions each organization is allowed to grant to its roles."""
__tablename__ = "org_permissions"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
org_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("orgs.uuid", ondelete="CASCADE")
)
permission_id: Mapped[str] = mapped_column(
String(64), ForeignKey("permissions.id", ondelete="CASCADE")
)
class RolePermission(Base):
"""Permissions that each role grants to its members."""
__tablename__ = "role_permissions"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
role_uuid: Mapped[bytes] = mapped_column(
LargeBinary(16), ForeignKey("roles.uuid", ondelete="CASCADE")
)
permission_id: Mapped[str] = mapped_column(
String(64), ForeignKey("permissions.id", ondelete="CASCADE")
)
class DB:
"""Legacy SQL database class for migration purposes only."""
def __init__(self, db_path: str = DB_PATH_DEFAULT):
"""Initialize with database path."""
self.engine = create_async_engine(db_path, echo=False)
# Ensure SQLite foreign key enforcement is ON for every new connection
if db_path.startswith("sqlite"):
@event.listens_for(self.engine.sync_engine, "connect")
def _fk_on(dbapi_connection, connection_record):
try:
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON;")
cursor.close()
except Exception:
pass
self.async_session_factory = async_sessionmaker(
self.engine, expire_on_commit=False
)
@asynccontextmanager
async def session(self):
"""Async context manager that provides a database session with transaction."""
async with self.async_session_factory() as session:
async with session.begin():
yield session
await session.flush()
await session.commit()
async def init_db(self) -> None:
"""Initialize database tables."""
async with self.engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
async def list_permissions(self) -> list[SqlPermission]:
async with self.session() as session:
result = await session.execute(select(PermissionModel))
return [p.as_dataclass() for p in result.scalars().all()]
async def list_organizations(self) -> list[_LegacyOrg]:
async with self.session() as session:
# Load all orgs
orgs_result = await session.execute(select(OrgModel))
org_models = orgs_result.scalars().all()
if not org_models:
return []
# Preload org permissions mapping
org_perms_result = await session.execute(select(OrgPermission))
org_perms = org_perms_result.scalars().all()
perms_by_org: dict[bytes, list[str]] = {}
for op in org_perms:
perms_by_org.setdefault(op.org_uuid, []).append(op.permission_id)
# Preload roles
roles_result = await session.execute(select(RoleModel))
role_models = roles_result.scalars().all()
# Preload role permissions mapping
rp_result = await session.execute(select(RolePermission))
rps = rp_result.scalars().all()
perms_by_role: dict[bytes, list[str]] = {}
for rp in rps:
perms_by_role.setdefault(rp.role_uuid, []).append(rp.permission_id)
# Build org dataclasses with roles and permission IDs
roles_by_org: dict[bytes, list[_LegacyRole]] = {}
for rm in role_models:
r_dc = rm.as_dataclass()
r_dc.permissions = perms_by_role.get(rm.uuid, [])
roles_by_org.setdefault(rm.org_uuid, []).append(r_dc)
orgs: list[_LegacyOrg] = []
for om in org_models:
o_dc = om.as_dataclass()
o_dc.permissions = perms_by_org.get(om.uuid, [])
o_dc.roles = roles_by_org.get(om.uuid, [])
orgs.append(o_dc)
return orgs
-5
View File
@@ -44,10 +44,6 @@ dev = [
"pytest-asyncio>=0.24.0", "pytest-asyncio>=0.24.0",
"httpx>=0.27.0", "httpx>=0.27.0",
] ]
migrate = [
"sqlalchemy[asyncio]>=2.0.0",
"aiosqlite>=0.19.0",
]
[tool.coverage.run] [tool.coverage.run]
source = ["paskia"] source = ["paskia"]
@@ -91,7 +87,6 @@ dev = [
[project.scripts] [project.scripts]
paskia = "paskia.fastapi.__main__:main" paskia = "paskia.fastapi.__main__:main"
paskia-migrate = "paskia.migrate:main"
[tool.hatch.build] [tool.hatch.build]
artifacts = ["paskia/frontend-build"] artifacts = ["paskia/frontend-build"]