"""Regression tests for runtime evidence propagation and report visibility. Covers: 1. Execution evidence is surfaced in reviewer/senior prompt context. 2. Reports include command preview and transcript excerpts. 3. Claude agentic failure detection (empty diff, write failure, expanded markers). 4. _format_execution_evidence produces expected output. """ from __future__ import annotations import tempfile import unittest from pathlib import Path from unittest.mock import MagicMock, patch from cross_eval.agent import ( AgentInvocationError, _claims_file_changes, _has_write_failure_indicators, invoke_agent_agentic, ) from cross_eval.config import BUILTIN_AGENTS from cross_eval.models import ( AgentConfig, AgentResult, IterationResult, PipelineConfig, PipelineResult, ReviewMetrics, StepConfig, ) from cross_eval.pipeline import _format_execution_evidence, run_pipeline from cross_eval.report import build_report # --------------------------------------------------------------------------- # 1. Execution evidence formatting # --------------------------------------------------------------------------- class TestFormatExecutionEvidence(unittest.TestCase): """_format_execution_evidence produces a compact summary for reviewers.""" def test_empty_results_returns_placeholder(self) -> None: self.assertIn("no prior execution evidence", _format_execution_evidence({})) def test_single_result_includes_key_fields(self) -> None: result = AgentResult( output="some diff", exit_code=0, agent_name="claude-coder", step_name="coding", duration_seconds=12.3, transcript="# Agent Execution Transcript\n\n## Command\nclaude ...", command_preview="claude --setting-sources user", ) evidence = _format_execution_evidence({"coding_output": result}) self.assertIn("claude-coder", evidence) self.assertIn("coding", evidence) self.assertIn("Exit code: 0", evidence) self.assertIn("12.3s", evidence) self.assertIn("claude --setting-sources user", evidence) self.assertIn("Transcript excerpt", evidence) def test_multiple_results_separated(self) -> None: r1 = AgentResult( output="diff1", exit_code=0, agent_name="coder", step_name="coding", duration_seconds=1.0, command_preview="cmd1", ) r2 = AgentResult( output="review text", exit_code=0, agent_name="reviewer", step_name="review", duration_seconds=2.0, command_preview="cmd2", ) evidence = _format_execution_evidence({ "coding_output": r1, "review_result": r2, }) self.assertIn("coder", evidence) self.assertIn("reviewer", evidence) self.assertIn("---", evidence) def test_transcript_truncated_at_2000_chars(self) -> None: long_transcript = "x" * 3000 result = AgentResult( output="out", exit_code=0, agent_name="agent", step_name="step", duration_seconds=1.0, transcript=long_transcript, ) evidence = _format_execution_evidence({"key": result}) self.assertIn("truncated", evidence) # The full 3000-char transcript should NOT appear self.assertNotIn("x" * 3000, evidence) # --------------------------------------------------------------------------- # 2. Evidence in reviewer prompts (integration) # --------------------------------------------------------------------------- class TestEvidenceInReviewerPrompt(unittest.TestCase): """Reviewer prompts include execution evidence from prior coding step.""" def test_reviewer_receives_evidence(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: steps = [ StepConfig( name="coding", agent="claude-coder", role="coding", prompt_template="default:coding", output_key="coding_output", ), StepConfig( name="review", agent="claude-reviewer", role="review", prompt_template="default:review", output_key="review_result", verdict=True, ), ] config = PipelineConfig( output_dir=Path(tmpdir), max_iterations=1, min_iterations=1, language="en", inputs={"plan": "Test plan", "checklist": "Test checklist"}, agents=dict(BUILTIN_AGENTS), coders=["claude-coder"], reviewers=["claude-reviewer"], pipeline=steps, preset_name="simple", ) captured_prompts: list[dict] = [] def _mock(agent_config, prompt, step_name, **kwargs): captured_prompts.append({ "step_name": step_name, "prompt": prompt, }) if step_name == "coding": return AgentResult( output="Implemented feature X", exit_code=0, agent_name=agent_config.name, step_name=step_name, duration_seconds=5.0, transcript="# Transcript\nclaude ran...", command_preview="claude --setting-sources user", ) return AgentResult( output="VERDICT: PASS", exit_code=0, agent_name=agent_config.name, step_name=step_name, duration_seconds=2.0, ) with patch("cross_eval.pipeline.invoke_agent", side_effect=_mock): result = run_pipeline(config) self.assertEqual(result.final_verdict, "PASS") # The reviewer prompt should contain execution evidence review_prompts = [ p for p in captured_prompts if p["step_name"] == "review" ] self.assertTrue(len(review_prompts) >= 1) review_prompt = review_prompts[0]["prompt"] # Evidence section should reference the coding step's command self.assertIn("Execution Evidence", review_prompt) self.assertIn("claude-coder", review_prompt) # --------------------------------------------------------------------------- # 3. Report includes evidence # --------------------------------------------------------------------------- class TestReportIncludesEvidence(unittest.TestCase): """Report generation includes command preview and transcript excerpts.""" def _make_pipeline_result(self) -> tuple[PipelineConfig, PipelineResult]: steps = [ StepConfig( name="coding", agent="claude-coder", role="coding", prompt_template="default:coding", output_key="coding_output", ), StepConfig( name="review", agent="claude-reviewer", role="review", prompt_template="default:review", output_key="review_result", verdict=True, ), ] config = PipelineConfig( max_iterations=1, language="en", inputs={"plan": "Plan", "checklist": "CL"}, agents=dict(BUILTIN_AGENTS), pipeline=steps, preset_name="simple", ) coding_result = AgentResult( output="diff --git a/file ...", exit_code=0, agent_name="claude-coder", step_name="coding", duration_seconds=10.0, transcript="# Agent Execution Transcript\n## Command\nclaude ...\n## Stdout\nok", command_preview="claude --setting-sources user", ) review_result = AgentResult( output="All good.\n\nVERDICT: PASS", exit_code=0, agent_name="claude-reviewer", step_name="review", duration_seconds=5.0, transcript="# Agent Execution Transcript\n## Command\nclaude -p ...\n## Stdout\nAll good.", command_preview="claude -p --setting-sources user", ) iteration = IterationResult( iteration=1, step_results={ "coding_output": coding_result, "review_result": review_result, }, step_outputs={ "coding_output": "diff --git a/file ...", "review_result": "All good.\n\nVERDICT: PASS", }, verdict="PASS", ) pipeline_result = PipelineResult( iterations=[iteration], final_verdict="PASS", total_duration=15.0, ) return config, pipeline_result def test_report_contains_command_preview(self) -> None: config, result = self._make_pipeline_result() report = build_report(config, result) self.assertIn("claude --setting-sources user", report) self.assertIn("**Command**", report) def test_report_contains_transcript_excerpt(self) -> None: config, result = self._make_pipeline_result() report = build_report(config, result) self.assertIn("Execution transcript", report) self.assertIn("Agent Execution Transcript", report) def test_report_contains_exit_code(self) -> None: config, result = self._make_pipeline_result() report = build_report(config, result) self.assertIn("**Exit code**: 0", report) # --------------------------------------------------------------------------- # 4. Claude agentic hardened failure detection # --------------------------------------------------------------------------- class TestClaimsFileChangesExpanded(unittest.TestCase): """Expanded change-claim markers detect more Claude output patterns.""" def test_ive_implemented(self) -> None: self.assertTrue(_claims_file_changes("I've implemented the feature")) def test_ive_updated(self) -> None: self.assertTrue(_claims_file_changes("I've updated the config")) def test_made_the_following_changes(self) -> None: self.assertTrue(_claims_file_changes("I made the following changes to the file")) def test_applied_the_fix(self) -> None: self.assertTrue(_claims_file_changes("Applied the fix for the bug")) def test_changes_have_been_applied(self) -> None: self.assertTrue(_claims_file_changes("Changes have been applied successfully")) def test_wrote_the_code(self) -> None: self.assertTrue(_claims_file_changes("Wrote the code for the new module")) def test_refactored(self) -> None: self.assertTrue(_claims_file_changes("I refactored the pipeline")) def test_no_changes_still_returns_false(self) -> None: self.assertFalse(_claims_file_changes("No changes were necessary")) def test_empty_string_returns_false(self) -> None: self.assertFalse(_claims_file_changes("")) class TestWriteFailureIndicators(unittest.TestCase): """_has_write_failure_indicators detects stderr patterns.""" def test_permission_denied(self) -> None: self.assertTrue(_has_write_failure_indicators("Error: Permission denied")) def test_read_only_filesystem(self) -> None: self.assertTrue(_has_write_failure_indicators("read-only file system")) def test_sandbox_restriction(self) -> None: self.assertTrue(_has_write_failure_indicators("Blocked by sandbox policy")) def test_eacces(self) -> None: self.assertTrue(_has_write_failure_indicators("EACCES: operation not permitted")) def test_empty_stderr_returns_false(self) -> None: self.assertFalse(_has_write_failure_indicators("")) def test_normal_stderr_returns_false(self) -> None: self.assertFalse(_has_write_failure_indicators("Downloading model...")) class TestAgenticWriteFailureRaisesError(unittest.TestCase): """Agentic mode raises AgentInvocationError on stderr write-failure indicators.""" @patch("cross_eval.worktree.capture_diff", return_value="") @patch("subprocess.run") def test_write_failure_detected_from_stderr( self, mock_run: MagicMock, mock_diff: MagicMock, ) -> None: mock_run.return_value = MagicMock( returncode=0, stdout="Done.", stderr="Error: Permission denied writing to /src/main.py", ) agent = AgentConfig( name="claude-coder", command="claude", args=["--setting-sources", "user"], agentic=True, ) import subprocess as _sp import tempfile as _tf with _tf.TemporaryDirectory() as td: wt = Path(td) _sp.run(["git", "init"], cwd=wt, capture_output=True, check=True) _sp.run(["git", "config", "user.email", "t@t.com"], cwd=wt, capture_output=True) _sp.run(["git", "config", "user.name", "T"], cwd=wt, capture_output=True) (wt / "README.md").write_text("# init\n") _sp.run(["git", "add", "."], cwd=wt, capture_output=True, check=True) _sp.run(["git", "commit", "-m", "init"], cwd=wt, capture_output=True, check=True) with self.assertRaises(AgentInvocationError) as ctx: invoke_agent_agentic( agent, "implement feature", "coding", worktree_path=wt, quiet=True, ) self.assertEqual(ctx.exception.failure_type, "WRITE_FAILURE") self.assertIn("Permission denied", ctx.exception.raw_error) class TestAgenticExpandedClaimMarkers(unittest.TestCase): """Agentic mode detects expanded claim markers in empty diff scenarios.""" @patch("cross_eval.worktree.capture_diff", return_value="") @patch("subprocess.run") def test_ive_implemented_triggers_empty_diff_error( self, mock_run: MagicMock, mock_diff: MagicMock, ) -> None: mock_run.return_value = MagicMock( returncode=0, stdout="I've implemented the requested changes to the pipeline.", stderr="", ) agent = AgentConfig( name="claude-coder", command="claude", args=["--setting-sources", "user"], agentic=True, ) import subprocess as _sp import tempfile as _tf with _tf.TemporaryDirectory() as td: wt = Path(td) _sp.run(["git", "init"], cwd=wt, capture_output=True, check=True) _sp.run(["git", "config", "user.email", "t@t.com"], cwd=wt, capture_output=True) _sp.run(["git", "config", "user.name", "T"], cwd=wt, capture_output=True) (wt / "README.md").write_text("# init\n") _sp.run(["git", "add", "."], cwd=wt, capture_output=True, check=True) _sp.run(["git", "commit", "-m", "init"], cwd=wt, capture_output=True, check=True) with self.assertRaises(AgentInvocationError) as ctx: invoke_agent_agentic( agent, "implement feature", "coding", worktree_path=wt, quiet=True, ) self.assertEqual(ctx.exception.failure_type, "EMPTY_DIFF") if __name__ == "__main__": unittest.main()