267 lines
9.5 KiB
Python
267 lines
9.5 KiB
Python
"""Tests for doctor, demo, and guided init features."""
|
|
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
from cross_eval.doctor import (
|
|
DoctorCheck,
|
|
check_cli_installed,
|
|
check_config,
|
|
format_doctor_results,
|
|
)
|
|
from cross_eval.demo import (
|
|
DEMO_CHECKLIST,
|
|
DEMO_PLAN,
|
|
run_mock_demo,
|
|
)
|
|
from cross_eval.cli import (
|
|
_generate_guided_config,
|
|
_prompt_choice,
|
|
_prompt_text,
|
|
main,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Doctor tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class DoctorCheckInstalledTest(unittest.TestCase):
|
|
def test_check_cli_installed_found(self) -> None:
|
|
with patch("cross_eval.doctor.shutil.which", return_value="/usr/bin/python3"):
|
|
with patch("cross_eval.doctor.subprocess.run") as mock_run:
|
|
mock_run.return_value = MagicMock(
|
|
stdout="Python 3.12.0", stderr=""
|
|
)
|
|
found, version = check_cli_installed("python3")
|
|
|
|
self.assertTrue(found)
|
|
self.assertIn("Python", version)
|
|
|
|
def test_check_cli_installed_not_found(self) -> None:
|
|
with patch("cross_eval.doctor.shutil.which", return_value=None):
|
|
found, msg = check_cli_installed("nonexistent-tool")
|
|
|
|
self.assertFalse(found)
|
|
self.assertIn("not found", msg)
|
|
|
|
def test_check_config_exists_valid(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
ce_dir = Path(tmpdir) / ".cross-eval"
|
|
ce_dir.mkdir()
|
|
config_path = ce_dir / "config.yaml"
|
|
config_path.write_text(
|
|
"inputs:\n plan: plan.md\ncoders: [claude-coder]\n"
|
|
"reviewers: [claude-reviewer]\npipeline: preset:simple\n",
|
|
encoding="utf-8",
|
|
)
|
|
# Also create plan.md so validation passes
|
|
(ce_dir / "plan.md").write_text("# Plan", encoding="utf-8")
|
|
|
|
ok, path, errors = check_config(Path(tmpdir))
|
|
|
|
self.assertTrue(ok)
|
|
self.assertIsNotNone(path)
|
|
self.assertEqual(errors, [])
|
|
|
|
def test_check_config_not_exists(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
ok, path, errors = check_config(Path(tmpdir))
|
|
|
|
self.assertFalse(ok)
|
|
self.assertIsNone(path)
|
|
|
|
def test_check_config_invalid(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
ce_dir = Path(tmpdir) / ".cross-eval"
|
|
ce_dir.mkdir()
|
|
# Valid YAML but missing required fields → validation fails
|
|
(ce_dir / "config.yaml").write_text(
|
|
"inputs:\n plan: /nonexistent/plan.md\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
ok, path, errors = check_config(Path(tmpdir))
|
|
|
|
self.assertFalse(ok)
|
|
self.assertIsNotNone(path)
|
|
|
|
def test_format_doctor_results_all_pass(self) -> None:
|
|
checks = [
|
|
DoctorCheck("test", True, True, "ok"),
|
|
DoctorCheck("test2", True, False, "ok"),
|
|
]
|
|
output = format_doctor_results(checks)
|
|
self.assertIn("✓", output)
|
|
self.assertIn("All checks passed", output)
|
|
|
|
def test_format_doctor_results_critical_fail(self) -> None:
|
|
checks = [
|
|
DoctorCheck("claude CLI", False, True, "not found"),
|
|
]
|
|
output = format_doctor_results(checks)
|
|
self.assertIn("✗", output)
|
|
self.assertIn("critical", output.lower())
|
|
|
|
def test_cmd_doctor_returns_0_all_pass(self) -> None:
|
|
with patch("cross_eval.doctor.run_doctor") as mock:
|
|
mock.return_value = [
|
|
DoctorCheck("test", True, True, "ok"),
|
|
]
|
|
exit_code = main(["doctor"])
|
|
self.assertEqual(exit_code, 0)
|
|
|
|
def test_cmd_doctor_returns_1_critical_fail(self) -> None:
|
|
with patch("cross_eval.doctor.run_doctor") as mock:
|
|
mock.return_value = [
|
|
DoctorCheck("claude CLI", False, True, "not found"),
|
|
]
|
|
exit_code = main(["doctor"])
|
|
self.assertEqual(exit_code, 1)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Demo tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class DemoTest(unittest.TestCase):
|
|
def test_demo_plan_is_nonempty(self) -> None:
|
|
self.assertIn("fibonacci", DEMO_PLAN.lower())
|
|
|
|
def test_demo_checklist_is_nonempty(self) -> None:
|
|
self.assertIn("fibonacci", DEMO_CHECKLIST.lower())
|
|
|
|
def test_mock_demo_runs_without_error(self) -> None:
|
|
# Should not raise
|
|
with patch("sys.stdout"):
|
|
run_mock_demo(preset="simple")
|
|
|
|
def test_mock_demo_escalate_runs_without_error(self) -> None:
|
|
with patch("sys.stdout"):
|
|
run_mock_demo(preset="simple", show_escalate=True)
|
|
|
|
def test_cmd_demo_mock_default(self) -> None:
|
|
with patch("cross_eval.demo.run_mock_demo") as mock:
|
|
exit_code = main(["demo"])
|
|
mock.assert_called_once_with(preset="simple", show_escalate=False)
|
|
self.assertEqual(exit_code, 0)
|
|
|
|
def test_cmd_demo_escalate_flag(self) -> None:
|
|
with patch("cross_eval.demo.run_mock_demo") as mock:
|
|
exit_code = main(["demo", "--escalate"])
|
|
mock.assert_called_once_with(preset="simple", show_escalate=True)
|
|
self.assertEqual(exit_code, 0)
|
|
|
|
def test_cmd_demo_live_requires_confirmation(self) -> None:
|
|
with patch("builtins.input", return_value="n"):
|
|
exit_code = main(["demo", "--live"])
|
|
self.assertEqual(exit_code, 0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Guided init tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class GuidedInitTest(unittest.TestCase):
|
|
def test_prompt_choice_default(self) -> None:
|
|
with patch("builtins.input", return_value=""):
|
|
result = _prompt_choice("Pick:", ["a", "b", "c"], default=2)
|
|
self.assertEqual(result, "b")
|
|
|
|
def test_prompt_choice_by_number(self) -> None:
|
|
with patch("builtins.input", return_value="3"):
|
|
result = _prompt_choice("Pick:", ["a", "b", "c"], default=1)
|
|
self.assertEqual(result, "c")
|
|
|
|
def test_prompt_choice_by_name(self) -> None:
|
|
with patch("builtins.input", return_value="simple"):
|
|
result = _prompt_choice("Pick:", ["simple", "review-fix"], default=1)
|
|
self.assertEqual(result, "simple")
|
|
|
|
def test_prompt_text_default(self) -> None:
|
|
with patch("builtins.input", return_value=""):
|
|
result = _prompt_text("Name", default="claude")
|
|
self.assertEqual(result, "claude")
|
|
|
|
def test_prompt_text_custom(self) -> None:
|
|
with patch("builtins.input", return_value="codex"):
|
|
result = _prompt_text("Name", default="claude")
|
|
self.assertEqual(result, "codex")
|
|
|
|
def test_generate_guided_config(self) -> None:
|
|
config = _generate_guided_config(
|
|
"review-fix", "ko",
|
|
{
|
|
"coder": "claude",
|
|
"reviewer": "codex",
|
|
"senior": "codex",
|
|
"max_iter": 5,
|
|
},
|
|
)
|
|
self.assertIn("preset:review-fix", config)
|
|
self.assertIn("language: ko", config)
|
|
self.assertIn("claude-coder", config)
|
|
self.assertIn("codex-reviewer", config)
|
|
self.assertIn("codex-senior", config)
|
|
self.assertIn("max_iterations: 5", config)
|
|
|
|
def test_generate_guided_config_full_name(self) -> None:
|
|
config = _generate_guided_config(
|
|
"simple", "ko",
|
|
{
|
|
"coder": "claude-coder",
|
|
"reviewer": "codex-reviewer",
|
|
"senior": "",
|
|
"max_iter": 3,
|
|
},
|
|
)
|
|
# Full names should not be double-suffixed
|
|
self.assertIn("claude-coder", config)
|
|
self.assertNotIn("claude-coder-coder", config)
|
|
self.assertIn("codex-reviewer", config)
|
|
self.assertNotIn("codex-reviewer-reviewer", config)
|
|
|
|
def test_generate_guided_config_no_senior(self) -> None:
|
|
config = _generate_guided_config(
|
|
"simple", "en",
|
|
{
|
|
"coder": "claude",
|
|
"reviewer": "claude",
|
|
"senior": "",
|
|
"max_iter": 3,
|
|
},
|
|
)
|
|
self.assertNotIn("senior", config.lower())
|
|
|
|
def test_guided_init_creates_files(self) -> None:
|
|
# Simulate guided init with all defaults
|
|
inputs = iter(["", "", "", "", "", "", ""])
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
with patch("builtins.input", side_effect=lambda _="": next(inputs, "")):
|
|
exit_code = main(["init", "--guided", "--dir", tmpdir])
|
|
|
|
config_path = Path(tmpdir) / ".cross-eval" / "config.yaml"
|
|
self.assertTrue(config_path.exists())
|
|
self.assertEqual(exit_code, 0)
|
|
|
|
def test_guided_init_preserves_existing_files(self) -> None:
|
|
inputs = iter(["", "", "", "", "", "", ""])
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
ce_dir = Path(tmpdir) / ".cross-eval"
|
|
ce_dir.mkdir()
|
|
existing = ce_dir / "config.yaml"
|
|
existing.write_text("# existing", encoding="utf-8")
|
|
|
|
with patch("builtins.input", side_effect=lambda _="": next(inputs, "")):
|
|
main(["init", "--guided", "--dir", tmpdir])
|
|
|
|
# Should not overwrite
|
|
self.assertEqual(existing.read_text(), "# existing")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|