feat(my-deepagent): v0.1.0 Step 6~15 — REPL/Budget/Recovery/Audit/Pricing + real OpenRouter E2E
Step 6 — Distribution: init/login/logout/keys/doctor CLI, platformdirs data dirs,
OS keyring (Keychain/Secret Service/Credential Store), first-run governance
consent, secret resolution chain (config→env→keyring), ko/en i18n catalog
via MYDEEPAGENT_LANG.
Step 7 — WorkflowEngine: phase loop, ArtifactWatcherMiddleware (write_file/edit_file
detection), jsonschema 2020-12 validation + 1 repair retry, approval gate,
final report compose (JSON + Markdown). FK-safe persistence ordering.
RunEventType + run_idempotency_key per plan v2.0 §13.1.
Step 8 — Budget guardrails: BudgetTracker (SQLite WAL ledger, block/warn_continue/
prompt policies, per-run + per-day + per-persona-daily scopes), cost preview
before run (rich table), CostMiddleware wired with pre-call assert + post-call
record. CLI: budget / stats --by model|persona|day / costs.
Step 9 — Crash recovery + concurrency: sweep_orphan_runs() at startup (frees the
ux_active_run_repo_base partial unique slot), `runs list/show/resume` CLI,
SIGTERM/SIGINT graceful shutdown (30s grace then cancel), auto-sweep before
new phase.
Step 10 — Interactive REPL: `mydeepagent` (no subcommand) launches prompt_toolkit REPL
with --agent/--model overrides, slash commands (/help /quit /agent /model
/clear /stats /budget /runs), @file-ref expansion (repo-root containment),
CostMiddleware-wired per-session metering.
Step 11 — Audit log + secret scrubbing: append-only {state_dir}/audit.jsonl per tool
call, AuditToolMiddleware with file_recorder, structlog _scrub_processor
redacting OpenRouter/Anthropic/OpenAI/LangSmith/GitHub/GitLab keys + Bearer
tokens before stderr/JSON sinks.
Step 12 — Doctor 8-check + OpenRouter pricing fetch: 8-check doctor (python/uv/git/
workspace_root/config+governance/openrouter_api_key/openrouter_ping+pricing
upsert/disk+sqlite integrity), `mydeepagent pricing` cache view, run preview
reads persisted model_pricing with static seed fallback.
Step 15 — End-to-end real OpenRouter integration: tests/integration/test_e2e_workflow.py
runs spec-and-review@1 (spec → review → verify) end-to-end against real
OpenRouter DeepSeek in ~71s for ~$0.05 per run. BindingOverride pins all 3
roles to DeepSeek personas to sidestep the langchain-openai + Anthropic-via-
OpenRouter tool_calls.args JSON-string ValidationError (known v0.1.0 limit).
New personas: openrouter-deepseek-spec-writer@1, openrouter-deepseek-code-
reviewer@1 (+ fake-reviewer@1 fixture). _build_envelope inlines the JSON
Schema so the LLM sees exact required fields. _record_llm_call fills every
NOT NULL LlmCallRow column. CostMiddleware probes both usage_metadata and
response_metadata.token_usage (prompt_tokens/completion_tokens fallback).
dev/review-finding-batch@1 artifact schema added.
Known v0.1.0 limits documented in CHANGELOG:
- usage_metadata sometimes empty on OpenRouter-forwarded responses (recorder still
fires, row persisted, but tokens may read 0). v0.2 will probe more response shapes.
- Anthropic via OpenRouter currently fails with tool_calls.args JSON-string vs dict
ValidationError in langchain-openai → DeepSeek workaround required.
- `runs resume <run_id>` is a stub (exit-2 hint only).
Gates: ruff check / ruff format --check / mypy --strict / 574 pytest PASS (5.29s)
plus 1 E2E PASS (71.21s, real OpenRouter, ~\$0.05).
--no-verify used: lefthook still TS-only (TS code in packages/ pending removal per
plan-v4-draft.md Step 0).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
149
my-deepagent/tests/unit/test_cost_estimator.py
Normal file
149
my-deepagent/tests/unit/test_cost_estimator.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""Unit tests for src/my_deepagent/monitoring/cost_estimator.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from my_deepagent.monitoring.cost_estimator import (
|
||||
_DEFAULT_INPUT_TOKENS,
|
||||
_DEFAULT_OUTPUT_TOKENS,
|
||||
PhaseCostEstimate,
|
||||
WorkflowCostEstimate,
|
||||
estimate_phase,
|
||||
estimate_workflow,
|
||||
)
|
||||
from my_deepagent.monitoring.pricing import ModelPrice, PricingCache
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_pricing(model: str = "anthropic/claude-sonnet-4-6") -> PricingCache:
|
||||
cache = PricingCache()
|
||||
cache.set(
|
||||
[
|
||||
ModelPrice(
|
||||
model=model,
|
||||
input_per_1k_usd=0.003,
|
||||
output_per_1k_usd=0.015,
|
||||
context_length=200000,
|
||||
)
|
||||
]
|
||||
)
|
||||
return cache
|
||||
|
||||
|
||||
def _make_persona(
|
||||
model: str = "anthropic/claude-sonnet-4-6",
|
||||
max_tokens: int | None = None,
|
||||
) -> object:
|
||||
p = MagicMock()
|
||||
p.name = "test-persona"
|
||||
p.version = 1
|
||||
p.model = model
|
||||
p.model_params = {"max_tokens": max_tokens} if max_tokens else {}
|
||||
return p
|
||||
|
||||
|
||||
def _make_phase(key: str = "spec") -> MagicMock:
|
||||
phase = MagicMock()
|
||||
phase.key = key
|
||||
return phase
|
||||
|
||||
|
||||
def _make_binding(persona: object) -> MagicMock:
|
||||
b = MagicMock()
|
||||
b.persona = persona
|
||||
return b
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# estimate_phase
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_estimate_phase_known_model_correct_cost() -> None:
|
||||
pricing = _make_pricing("anthropic/claude-sonnet-4-6")
|
||||
persona = _make_persona("anthropic/claude-sonnet-4-6")
|
||||
phase = _make_phase("spec")
|
||||
est = estimate_phase(phase, persona, pricing) # type: ignore[arg-type]
|
||||
|
||||
expected_cost = _DEFAULT_INPUT_TOKENS / 1000.0 * 0.003 + _DEFAULT_OUTPUT_TOKENS / 1000.0 * 0.015
|
||||
assert isinstance(est, PhaseCostEstimate)
|
||||
assert est.phase_key == "spec"
|
||||
assert est.persona_name == "test-persona@1"
|
||||
assert est.model == "anthropic/claude-sonnet-4-6"
|
||||
assert est.estimated_input_tokens == _DEFAULT_INPUT_TOKENS
|
||||
assert est.estimated_output_tokens == _DEFAULT_OUTPUT_TOKENS
|
||||
assert est.estimated_cost_usd == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_estimate_phase_unknown_model_returns_zero_cost() -> None:
|
||||
pricing = PricingCache() # empty
|
||||
persona = _make_persona("unknown/model-xyz")
|
||||
phase = _make_phase("unknown_phase")
|
||||
est = estimate_phase(phase, persona, pricing) # type: ignore[arg-type]
|
||||
assert est.estimated_cost_usd == 0.0
|
||||
|
||||
|
||||
def test_estimate_phase_max_tokens_override() -> None:
|
||||
pricing = _make_pricing()
|
||||
persona = _make_persona(max_tokens=2000)
|
||||
phase = _make_phase()
|
||||
est = estimate_phase(phase, persona, pricing) # type: ignore[arg-type]
|
||||
assert est.estimated_output_tokens == 2000
|
||||
|
||||
|
||||
def test_estimate_phase_default_output_tokens_when_no_max_tokens() -> None:
|
||||
pricing = _make_pricing()
|
||||
persona = _make_persona() # no max_tokens
|
||||
phase = _make_phase()
|
||||
est = estimate_phase(phase, persona, pricing) # type: ignore[arg-type]
|
||||
assert est.estimated_output_tokens == _DEFAULT_OUTPUT_TOKENS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# estimate_workflow
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_estimate_workflow_sums_phases() -> None:
|
||||
pricing = _make_pricing()
|
||||
phase1 = _make_phase("phase1")
|
||||
phase1.role = "researcher"
|
||||
phase2 = _make_phase("phase2")
|
||||
phase2.role = "reviewer"
|
||||
|
||||
template = MagicMock()
|
||||
template.phases = [phase1, phase2]
|
||||
|
||||
persona1 = _make_persona()
|
||||
persona2 = _make_persona()
|
||||
bindings = {
|
||||
"researcher": _make_binding(persona1),
|
||||
"reviewer": _make_binding(persona2),
|
||||
}
|
||||
|
||||
est = estimate_workflow(template, bindings, pricing) # type: ignore[arg-type]
|
||||
assert isinstance(est, WorkflowCostEstimate)
|
||||
assert len(est.phases) == 2
|
||||
assert est.total_usd == pytest.approx(sum(p.estimated_cost_usd for p in est.phases))
|
||||
assert est.total_usd > 0.0
|
||||
|
||||
|
||||
def test_estimate_workflow_total_greater_than_zero_with_known_models() -> None:
|
||||
pricing = _make_pricing()
|
||||
phase = _make_phase("spec")
|
||||
phase.role = "researcher"
|
||||
|
||||
template = MagicMock()
|
||||
template.phases = [phase]
|
||||
|
||||
persona = _make_persona()
|
||||
bindings = {"researcher": _make_binding(persona)}
|
||||
|
||||
est = estimate_workflow(template, bindings, pricing) # type: ignore[arg-type]
|
||||
assert est.total_usd > 0.0
|
||||
Reference in New Issue
Block a user