scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-10 22:43:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Record timeout and sandbox resources in scoreboard metadata

Browse source 
Adds --timeout, --sandbox-cpu, --sandbox-memory flags to record_results.py.
Re-recorded both existing runs with the new fields.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-16 11:24:56 -04:00
parent 5eccf9701b
commit 942054c43a
3 changed files with 23 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
evals/swe-bench/record_results.py
View file

@ -79,6 +79,18 @@ def main():
"--notes", default="",
help="Additional notes or observations",
)
parser.add_argument(
"--timeout", type=int, default=600,
help="Per-instance timeout used (seconds)",
)
parser.add_argument(
"--sandbox-cpu", type=int, default=2,
help="CPUs per Daytona sandbox",
)
parser.add_argument(
"--sandbox-memory", type=int, default=4,
help="Memory (GB) per Daytona sandbox",
)
args = parser.parse_args()
run_dir = SCOREBOARD_DIR / args.run_name
@ -167,6 +179,9 @@ def main():
"model": gen_summary.get("model", "unknown"),
"provider": gen_summary.get("provider", "unknown"),
"fabro_version": get_fabro_version(),
"timeout_s": args.timeout,
"sandbox_cpu": args.sandbox_cpu,
"sandbox_memory_gb": args.sandbox_memory,
"description": args.description,
"notes": args.notes,
"total_instances": total,

5
evals/swe-bench/scoreboard/haiku-baseline-20260316/meta.json
View file

@ -1,9 +1,12 @@
{
"run_name": "haiku-baseline-20260316",
"timestamp": "2026-03-16T13:53:04.401784+00:00",
"timestamp": "2026-03-16T15:24:29.679846+00:00",
"model": "claude-haiku-4-5",
"provider": "anthropic",
"fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)",
"timeout_s": 600,
"sandbox_cpu": 2,
"sandbox_memory_gb": 4,
"description": "Haiku 4.5 baseline, default prompt, 2 CPU / 4 GB, 10min timeout",
"notes": "",
"total_instances": 300,

5
evals/swe-bench/scoreboard/sonnet-baseline-20260316/meta.json
View file

@ -1,9 +1,12 @@
{
"run_name": "sonnet-baseline-20260316",
"timestamp": "2026-03-16T15:18:35.681947+00:00",
"timestamp": "2026-03-16T15:24:40.795288+00:00",
"model": "claude-sonnet-4-6",
"provider": "anthropic",
"fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)",
"timeout_s": 600,
"sandbox_cpu": 2,
"sandbox_memory_gb": 4,
"description": "Sonnet 4.6 baseline, default prompt, 2 CPU / 4 GB, 10min timeout",
"notes": "",
"total_instances": 300,