feat: ESCALATE verdict, issue tracker, onboarding commands
Add 3-verdict system (PASS/FAIL/ESCALATE) with priority handling across simple and phased pipelines. Senior reviewers can now escalate issues requiring human intervention, immediately breaking the review loop. - ESCALATE verdict extraction with highest priority over PASS/FAIL - Issue Tracker tables (ISS-NNN) carried across iterations - Auto-escalate heuristic using (file, keyword) composite fingerprints - Report restructuring: executive view first (verdict → tracker → metrics) - Onboarding: `doctor`, `demo`, `init --guided` commands - Exit codes: PASS=0, FAIL=1, ESCALATE=2 - 87 tests passing (54 config + 25 onboarding + 8 integration) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
chungyeong
267
tests/test_onboarding.py
Normal file
267
tests/test_onboarding.py
Normal file
@@ -0,0 +1,267 @@
|
||||
"""Tests for doctor, demo, and guided init features."""
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from cross_eval.doctor import (
|
||||
DoctorCheck,
|
||||
check_cli_installed,
|
||||
check_config,
|
||||
format_doctor_results,
|
||||
run_doctor,
|
||||
)
|
||||
from cross_eval.demo import (
|
||||
DEMO_CHECKLIST,
|
||||
DEMO_PLAN,
|
||||
run_mock_demo,
|
||||
)
|
||||
from cross_eval.cli import (
|
||||
_generate_guided_config,
|
||||
_prompt_choice,
|
||||
_prompt_text,
|
||||
main,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Doctor tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class DoctorCheckInstalledTest(unittest.TestCase):
|
||||
def test_check_cli_installed_found(self) -> None:
|
||||
with patch("cross_eval.doctor.shutil.which", return_value="/usr/bin/python3"):
|
||||
with patch("cross_eval.doctor.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
stdout="Python 3.12.0", stderr=""
|
||||
)
|
||||
found, version = check_cli_installed("python3")
|
||||
|
||||
self.assertTrue(found)
|
||||
self.assertIn("Python", version)
|
||||
|
||||
def test_check_cli_installed_not_found(self) -> None:
|
||||
with patch("cross_eval.doctor.shutil.which", return_value=None):
|
||||
found, msg = check_cli_installed("nonexistent-tool")
|
||||
|
||||
self.assertFalse(found)
|
||||
self.assertIn("not found", msg)
|
||||
|
||||
def test_check_config_exists_valid(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ce_dir = Path(tmpdir) / ".cross-eval"
|
||||
ce_dir.mkdir()
|
||||
config_path = ce_dir / "config.yaml"
|
||||
config_path.write_text(
|
||||
"inputs:\n plan: plan.md\ncoders: [claude-coder]\n"
|
||||
"reviewers: [claude-reviewer]\npipeline: preset:simple\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Also create plan.md so validation passes
|
||||
(ce_dir / "plan.md").write_text("# Plan", encoding="utf-8")
|
||||
|
||||
ok, path, errors = check_config(Path(tmpdir))
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIsNotNone(path)
|
||||
self.assertEqual(errors, [])
|
||||
|
||||
def test_check_config_not_exists(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ok, path, errors = check_config(Path(tmpdir))
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIsNone(path)
|
||||
|
||||
def test_check_config_invalid(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ce_dir = Path(tmpdir) / ".cross-eval"
|
||||
ce_dir.mkdir()
|
||||
# Valid YAML but missing required fields → validation fails
|
||||
(ce_dir / "config.yaml").write_text(
|
||||
"inputs:\n plan: /nonexistent/plan.md\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
ok, path, errors = check_config(Path(tmpdir))
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIsNotNone(path)
|
||||
|
||||
def test_format_doctor_results_all_pass(self) -> None:
|
||||
checks = [
|
||||
DoctorCheck("test", True, True, "ok"),
|
||||
DoctorCheck("test2", True, False, "ok"),
|
||||
]
|
||||
output = format_doctor_results(checks)
|
||||
self.assertIn("✓", output)
|
||||
self.assertIn("All checks passed", output)
|
||||
|
||||
def test_format_doctor_results_critical_fail(self) -> None:
|
||||
checks = [
|
||||
DoctorCheck("claude CLI", False, True, "not found"),
|
||||
]
|
||||
output = format_doctor_results(checks)
|
||||
self.assertIn("✗", output)
|
||||
self.assertIn("critical", output.lower())
|
||||
|
||||
def test_cmd_doctor_returns_0_all_pass(self) -> None:
|
||||
with patch("cross_eval.doctor.run_doctor") as mock:
|
||||
mock.return_value = [
|
||||
DoctorCheck("test", True, True, "ok"),
|
||||
]
|
||||
exit_code = main(["doctor"])
|
||||
self.assertEqual(exit_code, 0)
|
||||
|
||||
def test_cmd_doctor_returns_1_critical_fail(self) -> None:
|
||||
with patch("cross_eval.doctor.run_doctor") as mock:
|
||||
mock.return_value = [
|
||||
DoctorCheck("claude CLI", False, True, "not found"),
|
||||
]
|
||||
exit_code = main(["doctor"])
|
||||
self.assertEqual(exit_code, 1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Demo tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class DemoTest(unittest.TestCase):
|
||||
def test_demo_plan_is_nonempty(self) -> None:
|
||||
self.assertIn("fibonacci", DEMO_PLAN.lower())
|
||||
|
||||
def test_demo_checklist_is_nonempty(self) -> None:
|
||||
self.assertIn("fibonacci", DEMO_CHECKLIST.lower())
|
||||
|
||||
def test_mock_demo_runs_without_error(self) -> None:
|
||||
# Should not raise
|
||||
with patch("sys.stdout"):
|
||||
run_mock_demo(preset="simple")
|
||||
|
||||
def test_mock_demo_escalate_runs_without_error(self) -> None:
|
||||
with patch("sys.stdout"):
|
||||
run_mock_demo(preset="simple", show_escalate=True)
|
||||
|
||||
def test_cmd_demo_mock_default(self) -> None:
|
||||
with patch("cross_eval.demo.run_mock_demo") as mock:
|
||||
exit_code = main(["demo"])
|
||||
mock.assert_called_once_with(preset="simple", show_escalate=False)
|
||||
self.assertEqual(exit_code, 0)
|
||||
|
||||
def test_cmd_demo_escalate_flag(self) -> None:
|
||||
with patch("cross_eval.demo.run_mock_demo") as mock:
|
||||
exit_code = main(["demo", "--escalate"])
|
||||
mock.assert_called_once_with(preset="simple", show_escalate=True)
|
||||
self.assertEqual(exit_code, 0)
|
||||
|
||||
def test_cmd_demo_live_requires_confirmation(self) -> None:
|
||||
with patch("builtins.input", return_value="n"):
|
||||
exit_code = main(["demo", "--live"])
|
||||
self.assertEqual(exit_code, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Guided init tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class GuidedInitTest(unittest.TestCase):
|
||||
def test_prompt_choice_default(self) -> None:
|
||||
with patch("builtins.input", return_value=""):
|
||||
result = _prompt_choice("Pick:", ["a", "b", "c"], default=2)
|
||||
self.assertEqual(result, "b")
|
||||
|
||||
def test_prompt_choice_by_number(self) -> None:
|
||||
with patch("builtins.input", return_value="3"):
|
||||
result = _prompt_choice("Pick:", ["a", "b", "c"], default=1)
|
||||
self.assertEqual(result, "c")
|
||||
|
||||
def test_prompt_choice_by_name(self) -> None:
|
||||
with patch("builtins.input", return_value="simple"):
|
||||
result = _prompt_choice("Pick:", ["simple", "review-fix"], default=1)
|
||||
self.assertEqual(result, "simple")
|
||||
|
||||
def test_prompt_text_default(self) -> None:
|
||||
with patch("builtins.input", return_value=""):
|
||||
result = _prompt_text("Name", default="claude")
|
||||
self.assertEqual(result, "claude")
|
||||
|
||||
def test_prompt_text_custom(self) -> None:
|
||||
with patch("builtins.input", return_value="codex"):
|
||||
result = _prompt_text("Name", default="claude")
|
||||
self.assertEqual(result, "codex")
|
||||
|
||||
def test_generate_guided_config(self) -> None:
|
||||
config = _generate_guided_config(
|
||||
"review-fix", "ko",
|
||||
{
|
||||
"coder": "claude",
|
||||
"reviewer": "codex",
|
||||
"senior": "codex",
|
||||
"max_iter": 5,
|
||||
},
|
||||
)
|
||||
self.assertIn("preset:review-fix", config)
|
||||
self.assertIn("language: ko", config)
|
||||
self.assertIn("claude-coder", config)
|
||||
self.assertIn("codex-reviewer", config)
|
||||
self.assertIn("codex-senior", config)
|
||||
self.assertIn("max_iterations: 5", config)
|
||||
|
||||
def test_generate_guided_config_full_name(self) -> None:
|
||||
config = _generate_guided_config(
|
||||
"simple", "ko",
|
||||
{
|
||||
"coder": "claude-coder",
|
||||
"reviewer": "codex-reviewer",
|
||||
"senior": "",
|
||||
"max_iter": 3,
|
||||
},
|
||||
)
|
||||
# Full names should not be double-suffixed
|
||||
self.assertIn("claude-coder", config)
|
||||
self.assertNotIn("claude-coder-coder", config)
|
||||
self.assertIn("codex-reviewer", config)
|
||||
self.assertNotIn("codex-reviewer-reviewer", config)
|
||||
|
||||
def test_generate_guided_config_no_senior(self) -> None:
|
||||
config = _generate_guided_config(
|
||||
"simple", "en",
|
||||
{
|
||||
"coder": "claude",
|
||||
"reviewer": "claude",
|
||||
"senior": "",
|
||||
"max_iter": 3,
|
||||
},
|
||||
)
|
||||
self.assertNotIn("senior", config.lower())
|
||||
|
||||
def test_guided_init_creates_files(self) -> None:
|
||||
# Simulate guided init with all defaults
|
||||
inputs = iter(["", "", "", "", "", "", ""])
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
with patch("builtins.input", side_effect=lambda _="": next(inputs, "")):
|
||||
exit_code = main(["init", "--guided", "--dir", tmpdir])
|
||||
|
||||
config_path = Path(tmpdir) / ".cross-eval" / "config.yaml"
|
||||
self.assertTrue(config_path.exists())
|
||||
self.assertEqual(exit_code, 0)
|
||||
|
||||
def test_guided_init_preserves_existing_files(self) -> None:
|
||||
inputs = iter(["", "", "", "", "", "", ""])
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ce_dir = Path(tmpdir) / ".cross-eval"
|
||||
ce_dir.mkdir()
|
||||
existing = ce_dir / "config.yaml"
|
||||
existing.write_text("# existing", encoding="utf-8")
|
||||
|
||||
with patch("builtins.input", side_effect=lambda _="": next(inputs, "")):
|
||||
main(["init", "--guided", "--dir", tmpdir])
|
||||
|
||||
# Should not overwrite
|
||||
self.assertEqual(existing.read_text(), "# existing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user