chungyeong/cross-eval
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden runtime evidence and claude agentic validation

Browse Source
This commit is contained in:
chungyeong
2026-03-13 22:29:22 +09:00
parent 28dd794f54
commit 3fb19e90c0
5 changed files with 655 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
cross_eval/agent.py
View File

@@ -19,6 +19,34 @@ logger = logging.getLogger(__name__)
# CLI tools that support --system-prompt flag natively
_SYSTEM_PROMPT_AGENTS = ("claude",)
_REASONING_EFFORT_AGENTS = ("codex",)
_NO_CHANGE_ACK_MARKERS = (
"no changes",
"no code changes",
"no file changes",
"did not make any changes",
"nothing to change",
"no modifications were necessary",
"no update was necessary",
"already satisfied",
)
_CHANGE_CLAIM_MARKERS = (
"summary of all changes made",
"here's a summary of all changes made",
"implemented",
"i implemented",
"added",
"i added",
"updated",
"i updated",
"modified",
"i modified",
"created",
"i created",
"fixed",
"i fixed",
"completed the changes",
"finished the changes",
)
class AgentInvocationError(RuntimeError):
@@ -106,6 +134,16 @@ def _classify_agent_failure(detail: str) -> tuple[str, str]:
)
def _claims_file_changes(output: str) -> bool:
"""Heuristic for agent text that claims code changes were made."""
normalized = output.lower()
if not normalized.strip():
return False
if any(marker in normalized for marker in _NO_CHANGE_ACK_MARKERS):
return False
return any(marker in normalized for marker in _CHANGE_CLAIM_MARKERS)
class _Spinner:
"""Animated spinner for long-running agent calls."""
@@ -302,6 +340,9 @@ def invoke_agent(
command_preview=cmd_preview,
stdout=result.stdout,
stderr=result.stderr,
exit_code=result.returncode,
duration_seconds=round(duration, 1),
cwd=str(cwd) if cwd else "",
)
return AgentResult(
@@ -424,6 +465,28 @@ def invoke_agent_agentic(
diff_output = capture_diff(worktree_path)
if not diff_output:
stdout_excerpt = (result.stdout or "").strip()
stderr_excerpt = (result.stderr or "").strip()
if _claims_file_changes(stdout_excerpt):
if spinner:
spinner.stop(f"[{step_name}] FAILED (empty diff)")
raw_error = stdout_excerpt or "(stdout empty)"
if stderr_excerpt:
raw_error = f"{raw_error}\n\n[stderr]\n{stderr_excerpt}"
if len(raw_error) > 2000:
raw_error = raw_error[:2000] + "..."
raise AgentInvocationError(
agent_name=agent.name,
step_name=step_name,
cmd_preview=cmd_preview,
raw_error=raw_error,
failure_type="EMPTY_DIFF",
suggested_action=(
"Agent reported code changes but produced no git diff. "
"Treat this run as failed and require a real worktree diff before continuing."
),
)
diff_output = "(no changes)"
logger.warning(
"Agent '%s' made no file changes at step '%s'",
@@ -438,6 +501,9 @@ def invoke_agent_agentic(
command_preview=cmd_preview,
stdout=result.stdout,
stderr=result.stderr,
exit_code=result.returncode,
duration_seconds=round(duration, 1),
cwd=str(worktree_path),
)
return AgentResult(
@@ -456,6 +522,9 @@ def _build_transcript(
command_preview: str,
stdout: str,
stderr: str,
exit_code: int = 0,
duration_seconds: float = 0.0,
cwd: str = "",
) -> str:
"""Build a compact execution transcript for debugging/audit output."""
sections = [
@@ -466,6 +535,16 @@ def _build_transcript(
command_preview or "(unknown command)",
"```",
"",
]
if cwd:
sections.extend(["## Working Directory", f"`{cwd}`", ""])
sections.extend([
f"## Exit Code: {exit_code}",
"",
])
if duration_seconds > 0:
sections.extend([f"## Duration: {duration_seconds}s", ""])
sections.extend([
"## Stdout",
"```",
(stdout or "(empty)").strip(),
@@ -476,5 +555,5 @@ def _build_transcript(
(stderr or "(empty)").strip(),
"```",
"",
]
])
return "\n".join(sections)

275
cross_eval/discovery.py
View File

@@ -12,6 +12,7 @@ class RepoDiscovery:
package_managers: set[str] = field(default_factory=set)
databases: set[str] = field(default_factory=set)
services: set[str] = field(default_factory=set)
frameworks: set[str] = field(default_factory=set)
hints: list[str] = field(default_factory=list)
@@ -29,27 +30,176 @@ def _add_if_contains(target: set[str], content: str, mapping: dict[str, str]) ->
target.add(name)
# Shared mapping for database signals found in manifest content
_MANIFEST_DB_SIGNALS: dict[str, str] = {
# PostgreSQL
"psycopg": "postgresql",
"asyncpg": "postgresql",
"postgres": "postgresql",
"pgx": "postgresql",
# MySQL / MariaDB
"mysql": "mysql",
"mariadb": "mysql",
"pymysql": "mysql",
# MongoDB
"pymongo": "mongodb",
"mongodb": "mongodb",
"mongoengine": "mongodb",
"mongosh": "mongodb",
# ClickHouse
"clickhouse": "clickhouse",
"clickhouse-driver": "clickhouse",
"clickhouse_connect": "clickhouse",
# Redis
"redis": "redis",
"ioredis": "redis",
# SQLite
"sqlite": "sqlite",
"better-sqlite3": "sqlite",
"aiosqlite": "sqlite",
# Elasticsearch / OpenSearch
"elasticsearch": "elasticsearch",
"opensearch": "elasticsearch",
# DynamoDB
"dynamodb": "dynamodb",
"boto3": "dynamodb", # broad but common signal
# Cassandra
"cassandra-driver": "cassandra",
"cassandra": "cassandra",
# RabbitMQ
"amqplib": "rabbitmq",
"pika": "rabbitmq",
"rabbitmq": "rabbitmq",
# Kafka
"kafka": "kafka",
"confluent-kafka": "kafka",
"kafkajs": "kafka",
# Neo4j
"neo4j": "neo4j",
}
# Node package.json dependency → database mapping
_NODE_DEP_DB_SIGNALS: dict[str, str] = {
"pg": "postgresql",
"mysql": "mysql",
"mysql2": "mysql",
"mongoose": "mongodb",
"mongodb": "mongodb",
"@clickhouse/client": "clickhouse",
"redis": "redis",
"ioredis": "redis",
"prisma": "postgresql",
"better-sqlite3": "sqlite",
"sqlite3": "sqlite",
"@elastic/elasticsearch": "elasticsearch",
"@aws-sdk/client-dynamodb": "dynamodb",
"kafkajs": "kafka",
"amqplib": "rabbitmq",
"neo4j-driver": "neo4j",
"cassandra-driver": "cassandra",
"typeorm": "postgresql",
"sequelize": "postgresql",
"knex": "postgresql",
}
# Docker compose service image → service name mapping
_COMPOSE_SERVICE_SIGNALS: dict[str, str] = {
"clickhouse": "clickhouse",
"postgres": "postgresql",
"mysql": "mysql",
"mariadb": "mysql",
"mongo": "mongodb",
"redis": "redis",
"elasticsearch": "elasticsearch",
"opensearch": "elasticsearch",
"rabbitmq": "rabbitmq",
"kafka": "kafka",
"zookeeper": "kafka",
"cassandra": "cassandra",
"neo4j": "neo4j",
"minio": "s3",
"localstack": "aws-local",
"dynamodb": "dynamodb",
"memcached": "memcached",
"nginx": "nginx",
}
# Environment variable name patterns → database mapping
_ENV_DB_PATTERNS: list[tuple[str, str]] = [
("CLICKHOUSE", "clickhouse"),
("CH_", "clickhouse"),
("POSTGRES", "postgresql"),
("PG", "postgresql"),
("DATABASE_URL", "postgresql"),
("MYSQL", "mysql"),
("MARIADB", "mysql"),
("MONGO", "mongodb"),
("REDIS", "redis"),
("ELASTICSEARCH", "elasticsearch"),
("OPENSEARCH", "elasticsearch"),
("DYNAMO", "dynamodb"),
("CASSANDRA", "cassandra"),
("KAFKA", "kafka"),
("RABBIT", "rabbitmq"),
("AMQP", "rabbitmq"),
("NEO4J", "neo4j"),
("SQLITE", "sqlite"),
]
def discover_repo(project_root: Path, env_names: set[str] | None = None) -> RepoDiscovery:
"""Infer runtime-relevant stack hints from common manifest/config files."""
discovery = RepoDiscovery()
env_names = {name.upper() for name in (env_names or set())}
file_map = {
file_map: dict[str, Path] = {
"pyproject": project_root / "pyproject.toml",
"requirements": project_root / "requirements.txt",
"requirements_dev": project_root / "requirements-dev.txt",
"setup_py": project_root / "setup.py",
"setup_cfg": project_root / "setup.cfg",
"package": project_root / "package.json",
"go_mod": project_root / "go.mod",
"cargo": project_root / "Cargo.toml",
"gemfile": project_root / "Gemfile",
"build_gradle": project_root / "build.gradle",
"build_gradle_kts": project_root / "build.gradle.kts",
"pom": project_root / "pom.xml",
"composer": project_root / "composer.json",
"mix": project_root / "mix.exs",
"docker_compose": project_root / "docker-compose.yml",
"docker_compose_alt": project_root / "docker-compose.yaml",
"compose": project_root / "compose.yaml",
"prisma": project_root / "prisma" / "schema.prisma",
"dockerfile": project_root / "Dockerfile",
}
if file_map["pyproject"].exists() or file_map["requirements"].exists():
# ---- Language detection ----
if (
file_map["pyproject"].exists()
or file_map["requirements"].exists()
or file_map["requirements_dev"].exists()
or file_map["setup_py"].exists()
or file_map["setup_cfg"].exists()
):
discovery.languages.add("python")
if file_map["package"].exists():
discovery.languages.add("node")
if file_map["go_mod"].exists():
discovery.languages.add("go")
if file_map["cargo"].exists():
discovery.languages.add("rust")
if file_map["gemfile"].exists():
discovery.languages.add("ruby")
if file_map["build_gradle"].exists() or file_map["build_gradle_kts"].exists() or file_map["pom"].exists():
discovery.languages.add("java")
if file_map["composer"].exists():
discovery.languages.add("php")
if file_map["mix"].exists():
discovery.languages.add("elixir")
if file_map["pyproject"].exists():
# ---- Package manager detection ----
if file_map["pyproject"].exists() or file_map["requirements"].exists() or file_map["setup_py"].exists():
discovery.package_managers.add("pip")
if file_map["package"].exists():
try:
@@ -60,8 +210,29 @@ def discover_repo(project_root: Path, env_names: set[str] | None = None) -> Repo
if isinstance(pm, str) and pm:
discovery.package_managers.add(pm.split("@", 1)[0])
else:
discovery.package_managers.add("npm")
# Check for lockfiles to distinguish npm/yarn/pnpm
if (project_root / "pnpm-lock.yaml").exists():
discovery.package_managers.add("pnpm")
elif (project_root / "yarn.lock").exists():
discovery.package_managers.add("yarn")
else:
discovery.package_managers.add("npm")
if file_map["go_mod"].exists():
discovery.package_managers.add("go")
if file_map["cargo"].exists():
discovery.package_managers.add("cargo")
if file_map["gemfile"].exists():
discovery.package_managers.add("bundler")
if file_map["build_gradle"].exists() or file_map["build_gradle_kts"].exists():
discovery.package_managers.add("gradle")
if file_map["pom"].exists():
discovery.package_managers.add("maven")
if file_map["composer"].exists():
discovery.package_managers.add("composer")
if file_map["mix"].exists():
discovery.package_managers.add("mix")
# ---- Gather manifest content ----
manifests = {
name: _read_text(path)
for name, path in file_map.items()
@@ -69,24 +240,10 @@ def discover_repo(project_root: Path, env_names: set[str] | None = None) -> Repo
}
combined = "\n".join(manifests.values())
_add_if_contains(
discovery.databases,
combined,
{
"psycopg": "postgresql",
"asyncpg": "postgresql",
"postgres": "postgresql",
"mysql": "mysql",
"pymongo": "mongodb",
"mongodb": "mongodb",
"mongoengine": "mongodb",
"clickhouse": "clickhouse",
"clickhouse-driver": "clickhouse",
"clickhouse_connect": "clickhouse",
"redis": "redis",
},
)
# ---- Database detection from manifest content ----
_add_if_contains(discovery.databases, combined, _MANIFEST_DB_SIGNALS)
# ---- Node.js dependency-specific detection ----
if file_map["package"].exists():
try:
package_json = json.loads(_read_text(file_map["package"]) or "{}")
@@ -97,53 +254,57 @@ def discover_repo(project_root: Path, env_names: set[str] | None = None) -> Repo
**(package_json.get("devDependencies") or {}),
}
dep_blob = "\n".join(deps.keys()).lower()
_add_if_contains(
discovery.databases,
dep_blob,
{
"pg": "postgresql",
"mysql": "mysql",
"mongoose": "mongodb",
"mongodb": "mongodb",
"@clickhouse/client": "clickhouse",
"redis": "redis",
"prisma": "postgresql",
},
)
_add_if_contains(discovery.databases, dep_blob, _NODE_DEP_DB_SIGNALS)
# ---- Framework detection from manifest content ----
_add_if_contains(
discovery.frameworks,
combined,
{
"fastapi": "fastapi",
"django": "django",
"flask": "flask",
"express": "express",
"nextjs": "next.js",
"next": "next.js",
"nestjs": "nestjs",
"spring": "spring",
"rails": "rails",
"laravel": "laravel",
"phoenix": "phoenix",
"gin": "gin",
"actix": "actix",
},
)
# ---- Database detection from environment variable names ----
for env_name in env_names:
if "CLICKHOUSE" in env_name or env_name.startswith("CH_"):
discovery.databases.add("clickhouse")
if "POSTGRES" in env_name or env_name.startswith("PG") or env_name == "DATABASE_URL":
discovery.databases.add("postgresql")
if "MYSQL" in env_name:
discovery.databases.add("mysql")
if "MONGO" in env_name:
discovery.databases.add("mongodb")
if "REDIS" in env_name:
discovery.databases.add("redis")
for pattern, db_name in _ENV_DB_PATTERNS:
if pattern in env_name or env_name.startswith(pattern):
discovery.databases.add(db_name)
break
# ---- Docker compose service detection ----
compose_blob = "\n".join(
manifests.get(key, "")
for key in ("docker_compose", "docker_compose_alt", "compose")
).lower()
_add_if_contains(
discovery.services,
compose_blob,
{
"clickhouse": "clickhouse",
"postgres": "postgresql",
"mysql": "mysql",
"mongo": "mongodb",
"redis": "redis",
},
)
_add_if_contains(discovery.services, compose_blob, _COMPOSE_SERVICE_SIGNALS)
# ---- Hints from config files ----
if file_map["prisma"].exists():
discovery.hints.append("Prisma schema detected.")
if (project_root / "alembic.ini").exists():
discovery.hints.append("Alembic migration config detected.")
if (project_root / "docker").exists() or discovery.services:
if (project_root / "knexfile.js").exists() or (project_root / "knexfile.ts").exists():
discovery.hints.append("Knex migration config detected.")
if (project_root / "ormconfig.json").exists() or (project_root / "ormconfig.ts").exists():
discovery.hints.append("TypeORM config detected.")
if (project_root / "drizzle.config.ts").exists():
discovery.hints.append("Drizzle ORM config detected.")
if (project_root / "Makefile").exists():
discovery.hints.append("Makefile available for build/task automation.")
if file_map["dockerfile"].exists() or (project_root / "docker").exists() or discovery.services:
discovery.hints.append("Containerized services may be available for local verification.")
return discovery
@@ -160,6 +321,8 @@ def format_repo_discovery(discovery: RepoDiscovery) -> str:
lines.append("Detected databases/services in code or env: " + ", ".join(sorted(discovery.databases)))
if discovery.services:
lines.append("Detected local service containers: " + ", ".join(sorted(discovery.services)))
if discovery.frameworks:
lines.append("Detected frameworks: " + ", ".join(sorted(discovery.frameworks)))
if discovery.hints:
lines.extend(discovery.hints)
if not lines:

14
cross_eval/runtime_env.py
View File

@@ -14,9 +14,22 @@ _SUMMARY_PREFIXES = (
"PG",
"POSTGRES",
"MYSQL",
"MARIADB",
"REDIS",
"MONGO",
"ELASTICSEARCH",
"OPENSEARCH",
"DYNAMO",
"CASSANDRA",
"KAFKA",
"RABBIT",
"AMQP",
"NEO4J",
"SQLITE",
"MEMCACHED",
"AWS",
"S3",
"MINIO",
)
@@ -116,7 +129,6 @@ def summarize_environment(
key
for key in set(loaded_values) | set(env)
if key.startswith(_SUMMARY_PREFIXES)
or any(prefix in key for prefix in ("CLICKHOUSE", "DATABASE", "DB_"))
}
)
if visible_names: