diff --git a/eval/analysis/analyze_results.py b/eval/analysis/analyze_results.py index 69c5f5cdb..72a1fbbb7 100644 --- a/eval/analysis/analyze_results.py +++ b/eval/analysis/analyze_results.py @@ -19,6 +19,7 @@ import json import logging import os import subprocess +import sys from pathlib import Path from typing import Any @@ -164,7 +165,7 @@ def run_swebench_evaluation(results_dir: Path, run_id: str, subset: str = "lite" try: eval_output = results_dir / run_id / "swebench_eval" cmd = [ - "python", "-m", "swebench.harness.run_evaluation", + sys.executable, "-m", "swebench.harness.run_evaluation", "--dataset_name", dataset_mapping.get(subset, subset), "--predictions_path", str(preds_path), "--max_workers", "4", diff --git a/eval/configs/models/claude-haiku.yaml b/eval/configs/models/claude-haiku.yaml index 89746cdbd..548cc7f84 100644 --- a/eval/configs/models/claude-haiku.yaml +++ b/eval/configs/models/claude-haiku.yaml @@ -1,8 +1,7 @@ -# Claude 3.5 Haiku — fast, cheap, good baseline +# Claude Haiku 4.5 — fast, cheap, good baseline # Via OpenRouter (set OPENROUTER_API_KEY in .env) -# To use Anthropic directly, change to: anthropic/claude-3-5-haiku-20241022 model: - model_name: "openrouter/anthropic/claude-3.5-haiku" + model_name: "openrouter/anthropic/claude-haiku-4.5" cost_tracking: "ignore_errors" model_kwargs: max_tokens: 8192 diff --git a/eval/configs/models/minimax-m2.1.yaml b/eval/configs/models/minimax-m2.1.yaml new file mode 100644 index 000000000..766d75a00 --- /dev/null +++ b/eval/configs/models/minimax-m2.1.yaml @@ -0,0 +1,11 @@ +# MiniMax M2.5 — via OpenRouter (set OPENROUTER_API_KEY in .env) +# Uses text-based model class because MiniMax doesn't support tool_calls natively. +# The action_regex tells mini-swe-agent to parse ```bash blocks from responses. +model: + model_class: litellm_textbased + model_name: "openrouter/minimax/minimax-m2.5" + action_regex: "```(?:bash|mswea_bash_command)\\s*\\n(.*?)\\n```" + cost_tracking: "ignore_errors" + model_kwargs: + max_tokens: 8192 + temperature: 0 diff --git a/eval/pyproject.toml b/eval/pyproject.toml index ae9d2ad92..83ccb9416 100644 --- a/eval/pyproject.toml +++ b/eval/pyproject.toml @@ -30,6 +30,10 @@ gitnexus-eval-analyze = "analysis.analyze_results:app" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.build.targets.wheel] +packages = ["agents", "environments", "analysis", "bridge"] +extra-files = ["run_eval.py"] + [tool.ruff] line-length = 120 target-version = "py311" diff --git a/eval/run_eval.py b/eval/run_eval.py index 38dc7a473..7d410e7e4 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -178,7 +178,7 @@ def process_instance( env_class_name = env_config.pop("environment_class", "docker") if env_class_name == "eval.environments.gitnexus_docker.GitNexusDockerEnvironment": - from eval.environments.gitnexus_docker import GitNexusDockerEnvironment + from environments.gitnexus_docker import GitNexusDockerEnvironment env_config["image"] = get_swebench_docker_image(instance) env = GitNexusDockerEnvironment(**env_config) else: @@ -189,7 +189,7 @@ def process_instance( agent_config = dict(config.get("agent", {})) agent_class_name = agent_config.pop("agent_class", "eval.agents.gitnexus_agent.GitNexusAgent") - from eval.agents.gitnexus_agent import GitNexusAgent + from agents.gitnexus_agent import GitNexusAgent traj_path = instance_dir / f"{instance_id}.traj.json" agent_config["output_path"] = traj_path agent = GitNexusAgent(model, env, **agent_config) @@ -199,11 +199,18 @@ def process_instance( info = agent.run(instance["problem_statement"]) result["exit_status"] = info.get("exit_status") - result["submission"] = info.get("submission", "") result["cost"] = agent.cost result["n_calls"] = agent.n_calls result["gitnexus_metrics"] = agent.gitnexus_metrics.to_dict() + # Extract git diff patch from the container (SWE-bench needs the model_patch) + try: + patch_output = env.execute({"command": "cd /testbed && git diff"}) + result["submission"] = patch_output.get("output", "").strip() + except Exception as patch_err: + logger.warning(f"[{run_id}] Failed to extract patch: {patch_err}") + result["submission"] = info.get("submission", "") + except Exception as e: logger.error(f"[{run_id}] Error on {instance_id}: {e}") result["exit_status"] = type(e).__name__