fix(eval): import paths, patch extraction, model configs

- Fix imports from eval.agents/eval.environments to relative (agents/environments)
- Add hatch wheel config for correct package discovery
- Extract git diff patch from container for SWE-bench submission
- Use sys.executable instead of hardcoded "python" for venv compat
- Upgrade claude-haiku config to 4.5
- Add minimax-m2.1 model config

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
abhigyanpatwari 2026-02-25 17:11:00 +05:30
parent 91289404c2
commit 470a3377b3
5 changed files with 29 additions and 7 deletions

View file

@ -19,6 +19,7 @@ import json
import logging
import os
import subprocess
import sys
from pathlib import Path
from typing import Any
@ -164,7 +165,7 @@ def run_swebench_evaluation(results_dir: Path, run_id: str, subset: str = "lite"
try:
eval_output = results_dir / run_id / "swebench_eval"
cmd = [
"python", "-m", "swebench.harness.run_evaluation",
sys.executable, "-m", "swebench.harness.run_evaluation",
"--dataset_name", dataset_mapping.get(subset, subset),
"--predictions_path", str(preds_path),
"--max_workers", "4",

View file

@ -1,8 +1,7 @@
# Claude 3.5 Haiku — fast, cheap, good baseline
# Claude Haiku 4.5 — fast, cheap, good baseline
# Via OpenRouter (set OPENROUTER_API_KEY in .env)
# To use Anthropic directly, change to: anthropic/claude-3-5-haiku-20241022
model:
model_name: "openrouter/anthropic/claude-3.5-haiku"
model_name: "openrouter/anthropic/claude-haiku-4.5"
cost_tracking: "ignore_errors"
model_kwargs:
max_tokens: 8192

View file

@ -0,0 +1,11 @@
# MiniMax M2.5 — via OpenRouter (set OPENROUTER_API_KEY in .env)
# Uses text-based model class because MiniMax doesn't support tool_calls natively.
# The action_regex tells mini-swe-agent to parse ```bash blocks from responses.
model:
model_class: litellm_textbased
model_name: "openrouter/minimax/minimax-m2.5"
action_regex: "```(?:bash|mswea_bash_command)\\s*\\n(.*?)\\n```"
cost_tracking: "ignore_errors"
model_kwargs:
max_tokens: 8192
temperature: 0

View file

@ -30,6 +30,10 @@ gitnexus-eval-analyze = "analysis.analyze_results:app"
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["agents", "environments", "analysis", "bridge"]
extra-files = ["run_eval.py"]
[tool.ruff]
line-length = 120
target-version = "py311"

View file

@ -178,7 +178,7 @@ def process_instance(
env_class_name = env_config.pop("environment_class", "docker")
if env_class_name == "eval.environments.gitnexus_docker.GitNexusDockerEnvironment":
from eval.environments.gitnexus_docker import GitNexusDockerEnvironment
from environments.gitnexus_docker import GitNexusDockerEnvironment
env_config["image"] = get_swebench_docker_image(instance)
env = GitNexusDockerEnvironment(**env_config)
else:
@ -189,7 +189,7 @@ def process_instance(
agent_config = dict(config.get("agent", {}))
agent_class_name = agent_config.pop("agent_class", "eval.agents.gitnexus_agent.GitNexusAgent")
from eval.agents.gitnexus_agent import GitNexusAgent
from agents.gitnexus_agent import GitNexusAgent
traj_path = instance_dir / f"{instance_id}.traj.json"
agent_config["output_path"] = traj_path
agent = GitNexusAgent(model, env, **agent_config)
@ -199,11 +199,18 @@ def process_instance(
info = agent.run(instance["problem_statement"])
result["exit_status"] = info.get("exit_status")
result["submission"] = info.get("submission", "")
result["cost"] = agent.cost
result["n_calls"] = agent.n_calls
result["gitnexus_metrics"] = agent.gitnexus_metrics.to_dict()
# Extract git diff patch from the container (SWE-bench needs the model_patch)
try:
patch_output = env.execute({"command": "cd /testbed && git diff"})
result["submission"] = patch_output.get("output", "").strip()
except Exception as patch_err:
logger.warning(f"[{run_id}] Failed to extract patch: {patch_err}")
result["submission"] = info.get("submission", "")
except Exception as e:
logger.error(f"[{run_id}] Error on {instance_id}: {e}")
result["exit_status"] = type(e).__name__