mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-08-28 05:25:25 +00:00
fix(eval): import paths, patch extraction, model configs
- Fix imports from eval.agents/eval.environments to relative (agents/environments) - Add hatch wheel config for correct package discovery - Extract git diff patch from container for SWE-bench submission - Use sys.executable instead of hardcoded "python" for venv compat - Upgrade claude-haiku config to 4.5 - Add minimax-m2.1 model config Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
91289404c2
commit
470a3377b3
5 changed files with 29 additions and 7 deletions
|
|
@ -19,6 +19,7 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -164,7 +165,7 @@ def run_swebench_evaluation(results_dir: Path, run_id: str, subset: str = "lite"
|
|||
try:
|
||||
eval_output = results_dir / run_id / "swebench_eval"
|
||||
cmd = [
|
||||
"python", "-m", "swebench.harness.run_evaluation",
|
||||
sys.executable, "-m", "swebench.harness.run_evaluation",
|
||||
"--dataset_name", dataset_mapping.get(subset, subset),
|
||||
"--predictions_path", str(preds_path),
|
||||
"--max_workers", "4",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
# Claude 3.5 Haiku — fast, cheap, good baseline
|
||||
# Claude Haiku 4.5 — fast, cheap, good baseline
|
||||
# Via OpenRouter (set OPENROUTER_API_KEY in .env)
|
||||
# To use Anthropic directly, change to: anthropic/claude-3-5-haiku-20241022
|
||||
model:
|
||||
model_name: "openrouter/anthropic/claude-3.5-haiku"
|
||||
model_name: "openrouter/anthropic/claude-haiku-4.5"
|
||||
cost_tracking: "ignore_errors"
|
||||
model_kwargs:
|
||||
max_tokens: 8192
|
||||
|
|
|
|||
11
eval/configs/models/minimax-m2.1.yaml
Normal file
11
eval/configs/models/minimax-m2.1.yaml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# MiniMax M2.5 — via OpenRouter (set OPENROUTER_API_KEY in .env)
|
||||
# Uses text-based model class because MiniMax doesn't support tool_calls natively.
|
||||
# The action_regex tells mini-swe-agent to parse ```bash blocks from responses.
|
||||
model:
|
||||
model_class: litellm_textbased
|
||||
model_name: "openrouter/minimax/minimax-m2.5"
|
||||
action_regex: "```(?:bash|mswea_bash_command)\\s*\\n(.*?)\\n```"
|
||||
cost_tracking: "ignore_errors"
|
||||
model_kwargs:
|
||||
max_tokens: 8192
|
||||
temperature: 0
|
||||
|
|
@ -30,6 +30,10 @@ gitnexus-eval-analyze = "analysis.analyze_results:app"
|
|||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["agents", "environments", "analysis", "bridge"]
|
||||
extra-files = ["run_eval.py"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 120
|
||||
target-version = "py311"
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ def process_instance(
|
|||
env_class_name = env_config.pop("environment_class", "docker")
|
||||
|
||||
if env_class_name == "eval.environments.gitnexus_docker.GitNexusDockerEnvironment":
|
||||
from eval.environments.gitnexus_docker import GitNexusDockerEnvironment
|
||||
from environments.gitnexus_docker import GitNexusDockerEnvironment
|
||||
env_config["image"] = get_swebench_docker_image(instance)
|
||||
env = GitNexusDockerEnvironment(**env_config)
|
||||
else:
|
||||
|
|
@ -189,7 +189,7 @@ def process_instance(
|
|||
agent_config = dict(config.get("agent", {}))
|
||||
agent_class_name = agent_config.pop("agent_class", "eval.agents.gitnexus_agent.GitNexusAgent")
|
||||
|
||||
from eval.agents.gitnexus_agent import GitNexusAgent
|
||||
from agents.gitnexus_agent import GitNexusAgent
|
||||
traj_path = instance_dir / f"{instance_id}.traj.json"
|
||||
agent_config["output_path"] = traj_path
|
||||
agent = GitNexusAgent(model, env, **agent_config)
|
||||
|
|
@ -199,11 +199,18 @@ def process_instance(
|
|||
info = agent.run(instance["problem_statement"])
|
||||
|
||||
result["exit_status"] = info.get("exit_status")
|
||||
result["submission"] = info.get("submission", "")
|
||||
result["cost"] = agent.cost
|
||||
result["n_calls"] = agent.n_calls
|
||||
result["gitnexus_metrics"] = agent.gitnexus_metrics.to_dict()
|
||||
|
||||
# Extract git diff patch from the container (SWE-bench needs the model_patch)
|
||||
try:
|
||||
patch_output = env.execute({"command": "cd /testbed && git diff"})
|
||||
result["submission"] = patch_output.get("output", "").strip()
|
||||
except Exception as patch_err:
|
||||
logger.warning(f"[{run_id}] Failed to extract patch: {patch_err}")
|
||||
result["submission"] = info.get("submission", "")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{run_id}] Error on {instance_id}: {e}")
|
||||
result["exit_status"] = type(e).__name__
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue