feat(my-deepagent): v0.2 PR #1 — Postgres migration (ahead of M8-Py FastAPI)
Switches the production backing store from SQLite to PostgreSQL 16, per DR-2.
The migration trigger is two concurrent writers on the my-deepagent ORM
tables — which first appears with FastAPI (M8-Py). Doing the cut now keeps
the surface area small while M8-Py is still planning.
Production deps: `asyncpg`, `psycopg[binary]`, `langgraph-checkpoint-postgres`.
Test deps: `aiosqlite` (the bulk of unit + integration tests stay on sqlite
tmp_path for speed; the E2E suite and the new checkpointer tests exercise
the live Postgres path).
Highlights
- `persistence/db.py`: dialect-aware connect listener. SQLite still gets
WAL + busy_timeout=5000 + foreign_keys=ON; Postgres gets `SET TIME ZONE 'UTC'`.
Added `Database.dialect_name` + `drop_schema` (test-only).
- `persistence/checkpointer.py`: SqliteSaver → AsyncPostgresSaver. API is
now async (`async with`) and takes a connection string. SQLAlchemy URL
prefixes (`+asyncpg`, `+psycopg`) are auto-stripped to a plain libpq DSN
(`_to_psycopg_dsn` helper, 4 unit tests).
- `persistence/upsert.py` (new): `insert_for(session)` — dialect-aware UPSERT
helper. Picks `postgresql.insert` or `sqlite.insert` based on the bound
engine. Replaces 5 hardcoded `sqlite_insert` call sites in `budget.py`,
`recovery.py`, `cli/doctor.py`.
- `persistence/models.py`: `RunRow` partial unique index declares both
`postgresql_where=` and `sqlite_where=` for cross-dialect correctness.
- `config.py`: default `database_url` now
`postgresql+asyncpg://devflow:devflow@localhost:55432/mydeepagent`. v3
`devflow` DB preserved untouched; v4 lives in a fresh `mydeepagent` DB.
- `cli/doctor.py` check 8: dialect-aware DB liveness probe. Postgres path
runs `SELECT 1` (pg_isready equivalent); SQLite keeps `PRAGMA integrity_check`.
- `alembic/env.py`: env-aware URL resolution (`MYDEEPAGENT_DATABASE_URL` >
`DATABASE_URL` > default). Async driver prefixes are mapped to the sync
equivalents alembic needs.
- `alembic/versions/9f2a6c79667e_v0_2_baseline_schema_postgres.py` (new):
fresh baseline autogenerated against live Postgres. Old SQLite migrations
(`79945fdc2649`, `839f2233e346`) deleted — v0.2 starts a clean history.
- `tests/conftest.py` (new): `pg_db_url` async fixture creates a fresh DB
per test against docker-compose `devflow-postgres` and drops it on
teardown after terminating lingering backends.
- `tests/integration/test_checkpointer.py`: rewritten for AsyncPostgresSaver
(4 pure DSN-converter unit tests + 3 async context-manager integration tests).
- `tests/integration/test_e2e_workflow.py`: switched to `pg_db_url`. Real
OpenRouter E2E now exercises the production Postgres path end-to-end.
Recovery
- Previous SQLite database at the platformdirs data_dir is NOT auto-migrated;
v0.1.0 was the only release that wrote to it. Set
`MYDEEPAGENT_DATABASE_URL=sqlite+aiosqlite:///<path>` to read it.
- The v3 `devflow` Postgres DB is preserved untouched (separate database
name); to inspect: `psql -h localhost -p 55432 -U devflow -d devflow`.
Gates
- ruff check + ruff format --check + mypy --strict: PASS (102 source files)
- pytest non-E2E: 576 PASS (5.46 s)
- pytest E2E real OpenRouter on Postgres: 1 PASS (122.93 s, ~$0.05/run)
--no-verify: lefthook still TS-only (deleted in 0e61b2d but still queryable
in git history).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,7 @@ Checks:
|
||||
5. config + governance consent
|
||||
6. OpenRouter API key reachable
|
||||
7. OpenRouter /models ping + pricing matrix upsert
|
||||
8. Disk free + SQLite integrity_check
|
||||
8. Disk free + DB liveness probe (pg `SELECT 1` / sqlite `PRAGMA integrity_check`)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -26,7 +26,6 @@ import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
|
||||
from ..config import Config, load_config
|
||||
from ..errors import MyDeepAgentError
|
||||
@@ -38,6 +37,7 @@ from ..monitoring.pricing import (
|
||||
)
|
||||
from ..persistence.db import Database
|
||||
from ..persistence.models import ModelPricingRow
|
||||
from ..persistence.upsert import insert_for
|
||||
from ..secrets import resolve_openrouter_api_key
|
||||
|
||||
_CONSOLE = Console()
|
||||
@@ -147,9 +147,10 @@ async def _upsert_pricing(config: Config, prices: list[ModelPrice]) -> None:
|
||||
now = datetime.now(UTC).isoformat(timespec="seconds")
|
||||
try:
|
||||
async with db.session() as s:
|
||||
insert = insert_for(s)
|
||||
for p in prices:
|
||||
stmt = (
|
||||
sqlite_insert(ModelPricingRow)
|
||||
insert(ModelPricingRow)
|
||||
.values(
|
||||
model=p.model,
|
||||
input_per_1k_usd=p.input_per_1k_usd,
|
||||
@@ -175,6 +176,12 @@ async def _upsert_pricing(config: Config, prices: list[ModelPrice]) -> None:
|
||||
|
||||
|
||||
async def _check_disk_and_db(config: Config) -> CheckResult:
|
||||
"""Disk free + DB liveness probe.
|
||||
|
||||
Postgres path: ``SELECT 1`` round-trip (pg_isready equivalent — proves
|
||||
network reachability, auth, and that the DB exists).
|
||||
SQLite path: ``PRAGMA integrity_check`` to detect corruption.
|
||||
"""
|
||||
usage = shutil.disk_usage(str(config.workspace_root))
|
||||
free_gb = usage.free / (1024**3)
|
||||
if free_gb < 2.0:
|
||||
@@ -185,15 +192,34 @@ async def _check_disk_and_db(config: Config) -> CheckResult:
|
||||
disk_status = "ok"
|
||||
|
||||
db = Database(config.database_url)
|
||||
await db.init_schema()
|
||||
db_detail = ""
|
||||
db_ok = False
|
||||
try:
|
||||
# init_schema is idempotent and safe; for Postgres it requires CREATE
|
||||
# privileges, which the default devflow role has on the mydeepagent DB.
|
||||
# If alembic has already been applied this is a no-op.
|
||||
await db.init_schema()
|
||||
async with db.session() as s:
|
||||
row = (await s.execute(sa_text("PRAGMA integrity_check"))).scalar_one()
|
||||
if db.dialect_name == "postgresql":
|
||||
# pg_isready-equivalent: simple round-trip query proves the
|
||||
# server is reachable, auth works, and the DB exists.
|
||||
row = (await s.execute(sa_text("SELECT 1"))).scalar_one()
|
||||
db_ok = row == 1
|
||||
db_detail = f"postgres_alive={'ok' if db_ok else 'fail'}"
|
||||
elif db.dialect_name == "sqlite":
|
||||
row = (await s.execute(sa_text("PRAGMA integrity_check"))).scalar_one()
|
||||
db_ok = row == "ok"
|
||||
db_detail = f"sqlite_integrity={'ok' if db_ok else str(row)}"
|
||||
else: # pragma: no cover — defensive for future dialects
|
||||
db_ok = True
|
||||
db_detail = f"dialect={db.dialect_name},probe=skipped"
|
||||
except Exception as e:
|
||||
db_ok = False
|
||||
db_detail = f"db_error={type(e).__name__}:{e}"
|
||||
finally:
|
||||
await db.dispose()
|
||||
|
||||
db_ok = row == "ok"
|
||||
detail = f"free={free_gb:.1f}GB, sqlite_integrity={'ok' if db_ok else str(row)}"
|
||||
detail = f"free={free_gb:.1f}GB, {db_detail}"
|
||||
if disk_status == "fail" or not db_ok:
|
||||
final: Literal["ok", "warn", "fail"] = "fail"
|
||||
elif disk_status == "warn":
|
||||
|
||||
Reference in New Issue
Block a user