From 2436a58fcb14fba4bbbc012733e5dd6cd4c3ec1e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:43:51 +0000 Subject: [PATCH] chore(cookbook): profiling checkpoint after batch 10 Co-Authored-By: bot_apk --- .../capability_map_intermediate.json | 501 ++++++++++++++++++ .../checkpoints/batch_010/task_runs.jsonl | 93 ++++ 2 files changed, 594 insertions(+) create mode 100644 cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/capability_map_intermediate.json create mode 100644 cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/task_runs.jsonl diff --git a/cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/capability_map_intermediate.json b/cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/capability_map_intermediate.json new file mode 100644 index 00000000000..ebabaf563b4 --- /dev/null +++ b/cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/capability_map_intermediate.json @@ -0,0 +1,501 @@ +{ + "category_coverage": { + "algorithms_data": { + "profile_count": 0, + "status_counts": { + "evidenced_below_target": 0, + "insufficient_evidence": 0, + "no_evidence": 0, + "qualified": 0, + "unsupported_zero_success": 0 + } + }, + "feature_implementation": { + "profile_count": 0, + "status_counts": { + "evidenced_below_target": 0, + "insufficient_evidence": 0, + "no_evidence": 0, + "qualified": 0, + "unsupported_zero_success": 0 + } + }, + "repository_debugging": { + "profile_count": 3, + "status_counts": { + "evidenced_below_target": 3, + "insufficient_evidence": 0, + "no_evidence": 0, + "qualified": 0, + "unsupported_zero_success": 0 + } + }, + "shell_build": { + "profile_count": 0, + "status_counts": { + "evidenced_below_target": 0, + "insufficient_evidence": 0, + "no_evidence": 0, + "qualified": 0, + "unsupported_zero_success": 0 + } + }, + "unknown": { + "profile_count": 0, + "status_counts": { + "evidenced_below_target": 0, + "insufficient_evidence": 0, + "no_evidence": 0, + "qualified": 0, + "unsupported_zero_success": 0 + } + } + }, + "category_decisions": { + "algorithms_data": { + "decision": "unsupported_category", + "reason": "unsupported_category", + "selected_profile": null, + "threshold_unmet": false + }, + "feature_implementation": { + "decision": "no_qualified_model", + "reason": "no_qualified_model", + "selected_profile": null, + "threshold_unmet": false + }, + "repository_debugging": { + "decision": "no_qualified_model", + "reason": "no_qualified_model", + "selected_profile": null, + "threshold_unmet": false + }, + "shell_build": { + "decision": "unsupported_category", + "reason": "unsupported_category", + "selected_profile": null, + "threshold_unmet": false + }, + "unknown": { + "decision": "unsupported_category", + "reason": "unsupported_category", + "selected_profile": null, + "threshold_unmet": false + } + }, + "final": false, + "infra_deficits": [], + "interpretation": "Beta(1,1) credible bound describes the exchangeable public-dev task distribution for this category, not guaranteed per-task success", + "min_evidence_note": "necessary not sufficient: 13/13 is the minimum perfect cell clearing 0.80; not a general adequacy claim", + "preflight_failed": [ + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", + "latency_s": 36.985300702999666, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "haiku-4.5", + "sonnet-5", + "opus-5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-1413", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", + "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460" + }, + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest", + "latency_s": 45.86221657400165, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "haiku-4.5", + "sonnet-5", + "opus-5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-1439", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517", + "upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517" + }, + { + "agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", + "latency_s": 59.88555217099929, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "haiku-4.5", + "sonnet-5", + "opus-5" + ], + "noop_status": "unresolved", + "overlay_image_id": null, + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pyvista__pyvista-4329", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", + "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3" + }, + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", + "latency_s": 37.34210575900215, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "sonnet-5", + "opus-5", + "haiku-4.5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-1720", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", + "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337" + }, + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-995", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-995:latest", + "latency_s": 37.53449416300282, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "sonnet-5", + "opus-5", + "haiku-4.5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-995", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39", + "upstream_image_id": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39" + }, + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", + "latency_s": 40.2318156500005, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "haiku-4.5", + "sonnet-5", + "opus-5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-1334", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", + "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d" + }, + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", + "latency_s": 41.60962831600045, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "sonnet-5", + "opus-5", + "haiku-4.5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-1365", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", + "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2" + }, + { + "agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", + "latency_s": 53.46030479900219, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "haiku-4.5", + "sonnet-5", + "opus-5" + ], + "noop_status": "unresolved", + "overlay_image_id": null, + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pyvista__pyvista-4414", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", + "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92" + }, + { + "agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048", + "category": "repository_debugging", + "cost_usd": 0.0, + "error_class": "env", + "final_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc", + "gold_status": "unresolved", + "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", + "latency_s": 39.439905870000075, + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_order": [ + "sonnet-5", + "opus-5", + "haiku-4.5" + ], + "noop_status": "unresolved", + "overlay_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc", + "record_type": "preflight_failed", + "status": "preflight_failed", + "task_id": "pydicom__pydicom-1048", + "tenant_id": "public_dev_poc", + "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", + "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f" + } + ], + "preflight_incomplete": [ + "pyvista__pyvista-3675", + "pvlib__pvlib-python-1469", + "sqlfluff__sqlfluff-905", + "pvlib__pvlib-python-1186", + "pvlib__pvlib-python-1764", + "pvlib__pvlib-python-1073", + "pvlib__pvlib-python-1852", + "marshmallow-code__marshmallow-1702", + "pvlib__pvlib-python-907", + "pyvista__pyvista-432", + "pydicom__pydicom-1017", + "pydicom__pydicom-1241", + "pydicom__pydicom-1428", + "pydicom__pydicom-1228", + "pvlib__pvlib-python-718", + "pvlib__pvlib-python-1176", + "pvlib__pvlib-python-1181", + "pylint-dev__astroid-1616", + "pvlib__pvlib-python-823", + "pvlib__pvlib-python-1478", + "pylint-dev__astroid-1417", + "pvlib__pvlib-python-1682", + "pyvista__pyvista-4648", + "pyvista__pyvista-4311", + "pvlib__pvlib-python-1426", + "marshmallow-code__marshmallow-2123", + "pvlib__pvlib-python-1458", + "pvlib__pvlib-python-807", + "pvlib__pvlib-python-1395", + "pvlib__pvlib-python-1832" + ], + "profiles": [ + { + "attempt_count": 21, + "beta_mean": 0.34782608695652173, + "cost_per_graded_task_usd": 0.1902889880952381, + "graded_count": 21, + "infra_count": 0, + "key": { + "budget_version": "budget_1usd_50_600_v1", + "category": "repository_debugging", + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", + "harness_version": "run_pilot_attempt_v1", + "model_alias": "haiku-4.5", + "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", + "tenant_id": "public_dev_poc" + }, + "lower_5pct": 0.19556261654815532, + "mean_latency_s": 156.67234382552382, + "min_evidence": 13, + "primary_labels": [ + "repository_debugging" + ], + "provenance": { + "alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated", + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_alias": "haiku-4.5" + }, + "raw_rate": 0.3333333333333333, + "secondary_labels": [], + "status": "evidenced_below_target", + "successes": 7, + "tau": 0.8, + "total_spend_usd": 3.99606875 + }, + { + "attempt_count": 21, + "beta_mean": 0.391304347826087, + "cost_per_graded_task_usd": 0.5018432380952381, + "graded_count": 21, + "infra_count": 0, + "key": { + "budget_version": "budget_1usd_50_600_v1", + "category": "repository_debugging", + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", + "harness_version": "run_pilot_attempt_v1", + "model_alias": "opus-5", + "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", + "tenant_id": "public_dev_poc" + }, + "lower_5pct": 0.23272387867600225, + "mean_latency_s": 175.2440282505245, + "min_evidence": 13, + "primary_labels": [ + "repository_debugging" + ], + "provenance": { + "alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated", + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_alias": "opus-5" + }, + "raw_rate": 0.38095238095238093, + "secondary_labels": [], + "status": "evidenced_below_target", + "successes": 8, + "tau": 0.8, + "total_spend_usd": 10.538708 + }, + { + "attempt_count": 21, + "beta_mean": 0.5217391304347826, + "cost_per_graded_task_usd": 0.25978090000000004, + "graded_count": 21, + "infra_count": 0, + "key": { + "budget_version": "budget_1usd_50_600_v1", + "category": "repository_debugging", + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", + "harness_version": "run_pilot_attempt_v1", + "model_alias": "sonnet-5", + "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", + "tenant_id": "public_dev_poc" + }, + "lower_5pct": 0.3525440815834854, + "mean_latency_s": 159.05449886028578, + "min_evidence": 13, + "primary_labels": [ + "repository_debugging" + ], + "provenance": { + "alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated", + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", + "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", + "model_alias": "sonnet-5" + }, + "raw_rate": 0.5238095238095238, + "secondary_labels": [], + "status": "evidenced_below_target", + "successes": 11, + "tau": 0.8, + "total_spend_usd": 5.4553989000000005 + } + ], + "quarantined_attempts": [], + "tenant_id": "public_dev_poc", + "thresholds": { + "min_evidence": 13, + "tau": 0.8 + }, + "versions": { + "budget_version": "budget_1usd_50_600_v1", + "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", + "harness_version": "run_pilot_attempt_v1", + "model_deployments": { + "haiku-4.5": [ + "13ef3d99-7edf-4a6b-aab6-f088c8e25964" + ], + "opus-5": [ + "13ef3d99-7edf-4a6b-aab6-f088c8e25964" + ], + "sonnet-5": [ + "13ef3d99-7edf-4a6b-aab6-f088c8e25964" + ] + }, + "model_versions": { + "haiku-4.5": [ + "litellm_proxy/anthropic/claude-haiku-4-5-20251001" + ], + "opus-5": [ + "litellm_proxy/anthropic/claude-opus-5" + ], + "sonnet-5": [ + "litellm_proxy/anthropic/claude-sonnet-5" + ] + }, + "observed_model_cohorts": [ + { + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "model_alias": "haiku-4.5", + "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001" + }, + { + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "model_alias": "opus-5", + "returned_model_version": "litellm_proxy/anthropic/claude-opus-5" + }, + { + "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", + "model_alias": "sonnet-5", + "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5" + } + ], + "version_changes": [] + } +} diff --git a/cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/task_runs.jsonl b/cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/task_runs.jsonl new file mode 100644 index 00000000000..6879fd85de5 --- /dev/null +++ b/cookbook/capability_routing_pilot/profiling/checkpoints/batch_010/task_runs.jsonl @@ -0,0 +1,93 @@ +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "latency_s": 36.985300702999666, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "latency_s": 37.443159876998834, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "latency_s": 42.63817018199916, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15246375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.22013637100099, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:29:42.514070+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.00210723499913, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 11, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1216__opus-5__a1__20261021T042841152530Z", "solve_latency_s": 40.00210723499913, "started_at": "2026-10-21T04:28:41.152510+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11155, "cache_read": 72220, "completion": 1861, "prompt": 83397, "reasoning": 34}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0754433, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.261532576998434, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:29:49.569191+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.00336846000209, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 15, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1192__sonnet-5__a1__20261021T042841151566Z", "solve_latency_s": 50.00336846000209, "started_at": "2026-10-21T04:28:41.151547+00:00", "success": true, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11487, "cache_read": 101329, "completion": 2640, "prompt": 112846, "reasoning": 66}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.08664235, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.42284784900039, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:31:34.083967+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00328306499796, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 25, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1216__haiku-4.5__a1__20261021T042942515886Z", "solve_latency_s": 90.00328306499796, "started_at": "2026-10-21T04:29:42.515874+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17291, "cache_read": 192516, "completion": 6628, "prompt": 222444, "reasoning": 0}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.38297225, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 20.581349566000426, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:32:00.355179+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.00375107299988, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 20, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1192__opus-5__a1__20261021T042949571022Z", "solve_latency_s": 110.00375107299988, "started_at": "2026-10-21T04:29:49.571011+00:00", "success": true, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17105, "cache_read": 226732, "completion": 6500, "prompt": 243877, "reasoning": 3366}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0498986, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.37230331899991, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:32:35.584861+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.00220326099952, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1216__sonnet-5__a1__20261021T043134086373Z", "solve_latency_s": 40.00220326099952, "started_at": "2026-10-21T04:31:34.086359+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8744, "cache_read": 53193, "completion": 1736, "prompt": 61957, "reasoning": 10}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2071005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:34:50.522036+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.0046138639991, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1192__haiku-4.5__a1__20261021T043200357212Z", "solve_latency_s": 170.0046138639991, "started_at": "2026-10-21T04:32:00.357202+00:00", "success": false, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31386, "cache_read": 742450, "completion": 15459, "prompt": 790164, "reasoning": 0}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest", "latency_s": 45.86221657400165, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1439", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517", "upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "latency_s": 46.50070145599966, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "latency_s": 39.32556367899815, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6126465, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:47:21.107529+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00617045400213, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 27, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-2307__opus-5__a1__20261021T044400932430Z", "solve_latency_s": 200.00617045400213, "started_at": "2026-10-21T04:44:00.932414+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23326, "cache_read": 365428, "completion": 11355, "prompt": 388808, "reasoning": 6719}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3954644, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 22.276458722000825, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:48:33.342443+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.0124097070002, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 32, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1165__sonnet-5__a1__20261021T044400931150Z", "solve_latency_s": 250.0124097070002, "started_at": "2026-10-21T04:44:00.931128+00:00", "success": true, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30230, "cache_read": 513457, "completion": 21707, "prompt": 543751, "reasoning": 15209}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1836434, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:50:01.255188+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00828153600014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-2307__haiku-4.5__a1__20261021T044721112372Z", "solve_latency_s": 160.00828153600014, "started_at": "2026-10-21T04:47:21.112358+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 29088, "cache_read": 697624, "completion": 12455, "prompt": 741958, "reasoning": 0}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.626164, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:51:33.536769+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00563653500285, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 13, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1165__opus-5__a1__20261021T044833345077Z", "solve_latency_s": 180.00563653500285, "started_at": "2026-10-21T04:48:33.345065+00:00", "success": false, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24668, "cache_read": 184418, "completion": 15186, "prompt": 209112, "reasoning": 12703}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.26214350000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:55:23.672687+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00504096200166, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1165__haiku-4.5__a1__20261021T045133539818Z", "solve_latency_s": 230.00504096200166, "started_at": "2026-10-21T04:51:33.539806+00:00", "success": false, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 43394, "cache_read": 976470, "completion": 19432, "prompt": 1032958, "reasoning": 0}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5375916000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:55:41.469860+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 340.0079844879983, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-2307__sonnet-5__a1__20261021T045001257820Z", "solve_latency_s": 340.0079844879983, "started_at": "2026-10-21T04:50:01.257807+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37278, "cache_read": 973333, "completion": 24953, "prompt": 1010711, "reasoning": 15582}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "latency_s": 59.88555217099929, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "latency_s": 37.34210575900215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "latency_s": 36.9401233719982, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.49552525000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.36470413599818, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:01:43.053923+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00416897800096, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 19, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-897__opus-5__a1__20261021T045854563009Z", "solve_latency_s": 150.00416897800096, "started_at": "2026-10-21T04:58:54.562993+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19647, "cache_read": 236183, "completion": 10178, "prompt": 255868, "reasoning": 5998}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15681435, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:03:53.181859+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.0038786289988, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-897__haiku-4.5__a1__20261021T050143057007Z", "solve_latency_s": 130.0038786289988, "started_at": "2026-10-21T05:01:43.056996+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23533, "cache_read": 670341, "completion": 9320, "prompt": 707638, "reasoning": 0}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2344321, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.46944500400059, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:06:41.825678+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00388437999936, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 45, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-897__sonnet-5__a1__20261021T050353184991Z", "solve_latency_s": 150.00388437999936, "started_at": "2026-10-21T05:03:53.184978+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18927, "cache_read": 459623, "completion": 9501, "prompt": 478640, "reasoning": 3384}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "latency_s": 39.889730383001734, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-995", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-995:latest", "latency_s": 37.53449416300282, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-995", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39", "upstream_image_id": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "latency_s": 39.2809497240014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.32255725, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.878957515000366, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:11:02.894305+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 80.00347018200046, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 13, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1349__opus-5__a1__20261021T050922867062Z", "solve_latency_s": 80.00347018200046, "started_at": "2026-10-21T05:09:22.867049+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18737, "cache_read": 157992, "completion": 5053, "prompt": 176755, "reasoning": 2608}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1955686, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:11:23.004924+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00499666500036, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1364__haiku-4.5__a1__20261021T050922865588Z", "solve_latency_s": 120.00499666500036, "started_at": "2026-10-21T05:09:22.865569+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37378, "cache_read": 1011631, "completion": 7784, "prompt": 1057772, "reasoning": 0}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.25508825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.928946245003317, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:14:22.849769+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00484361299823, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 46, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1349__haiku-4.5__a1__20261021T051102898250Z", "solve_latency_s": 180.00484361299823, "started_at": "2026-10-21T05:11:02.898236+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 42975, "cache_read": 1255085, "completion": 13855, "prompt": 1304646, "reasoning": 0}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1126159, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.824896968002577, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:15:32.696323+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.0022958030022, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 14, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1349__sonnet-5__a1__20261021T051422853521Z", "solve_latency_s": 50.0022958030022, "started_at": "2026-10-21T05:14:22.853508+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18231, "cache_read": 170812, "completion": 3282, "prompt": 189071, "reasoning": 872}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.4375135, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:16:03.478380+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.0084194079973, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1364__sonnet-5__a1__20261021T051123009015Z", "solve_latency_s": 280.0084194079973, "started_at": "2026-10-21T05:11:23.009002+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35875, "cache_read": 974680, "completion": 15269, "prompt": 1010655, "reasoning": 8390}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.648212, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:20:13.628921+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.0070123559999, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1364__opus-5__a1__20261021T051603482193Z", "solve_latency_s": 250.0070123559999, "started_at": "2026-10-21T05:16:03.482180+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19296, "cache_read": 169404, "completion": 17710, "prompt": 188732, "reasoning": 15644}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "latency_s": 39.77621123099743, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "latency_s": 43.17733980900084, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "latency_s": 37.75819920799768, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0995685, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.37739269600206, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:23:28.637543+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.001993197998672, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 6, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1164__opus-5__a1__20261021T052240132919Z", "solve_latency_s": 30.001993197998672, "started_at": "2026-10-21T05:22:40.132907+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 7776, "cache_read": 27567, "completion": 1485, "prompt": 35355, "reasoning": 253}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.08022180000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.130332180997357, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:23:51.415788+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.007217943002615, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1719__sonnet-5__a1__20261021T052240127823Z", "solve_latency_s": 50.007217943002615, "started_at": "2026-10-21T05:22:40.127805+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11136, "cache_read": 111339, "completion": 3005, "prompt": 122507, "reasoning": 523}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.18115045, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.322718805000477, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:25:09.592869+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00579304400162, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 46, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1810__haiku-4.5__a1__20261021T052240130927Z", "solve_latency_s": 130.00579304400162, "started_at": "2026-10-21T05:22:40.130915+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33073, "cache_read": 697242, "completion": 10692, "prompt": 746940, "reasoning": 0}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.12290365, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.234139322001283, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:25:27.046567+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00316618500074, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 39, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1164__haiku-4.5__a1__20261021T052328641925Z", "solve_latency_s": 100.00316618500074, "started_at": "2026-10-21T05:23:28.641910+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 21985, "cache_read": 420494, "completion": 7907, "prompt": 456317, "reasoning": 0}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.035914600000000005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.89811670599738, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:25:59.649361+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.00381014999948, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 9, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1810__sonnet-5__a1__20261021T052509597736Z", "solve_latency_s": 30.00381014999948, "started_at": "2026-10-21T05:25:09.597723+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 6304, "cache_read": 35943, "completion": 1293, "prompt": 42265, "reasoning": 40}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.031602, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.185294439001154, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:26:16.378423+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.00225875400065, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 8, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1164__sonnet-5__a1__20261021T052527051475Z", "solve_latency_s": 30.00225875400065, "started_at": "2026-10-21T05:25:27.051461+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 5698, "cache_read": 27125, "completion": 1190, "prompt": 32839, "reasoning": 61}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.23039375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.577401711001585, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:27:49.390345+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00354062100087, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 8, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1810__opus-5__a1__20261021T052559657755Z", "solve_latency_s": 90.00354062100087, "started_at": "2026-10-21T05:25:59.657740+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8455, "cache_read": 41590, "completion": 6267, "prompt": 50061, "reasoning": 4667}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.66210825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 23.277767100000347, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:28:14.846207+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.00587134300076, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 27, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1719__opus-5__a1__20261021T052351420018Z", "solve_latency_s": 240.00587134300076, "started_at": "2026-10-21T05:23:51.420006+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 20941, "cache_read": 368864, "completion": 13861, "prompt": 389859, "reasoning": 8383}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1483624, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.447099036999134, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:31:06.320651+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00421196300158, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 42, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1719__haiku-4.5__a1__20261021T052814852065Z", "solve_latency_s": 150.00421196300158, "started_at": "2026-10-21T05:28:14.852050+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25984, "cache_read": 629064, "completion": 9321, "prompt": 661419, "reasoning": 0}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "latency_s": 40.2318156500005, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1334", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "latency_s": 47.930545192000864, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "latency_s": 43.92638719699971, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.46326425000000004, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.742469348002487, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "finished_at": "2026-10-21T05:36:48.224174+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.0067291660016, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 16, "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-980__opus-5__a1__20261021T053406317144Z", "solve_latency_s": 140.0067291660016, "started_at": "2026-10-21T05:34:06.317128+00:00", "success": false, "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22519, "cache_read": 216521, "completion": 8564, "prompt": 239072, "reasoning": 5572}, "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.30063480000000004, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "finished_at": "2026-10-21T05:37:16.454317+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0117116170004, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1351__sonnet-5__a1__20261021T053406312824Z", "solve_latency_s": 190.0117116170004, "started_at": "2026-10-21T05:34:06.312804+00:00", "success": false, "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19556, "cache_read": 538174, "completion": 14391, "prompt": 557830, "reasoning": 8757}, "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1637955, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.43748659799894, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "finished_at": "2026-10-21T05:39:39.812688+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00504884099792, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 34, "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-980__haiku-4.5__a1__20261021T053648229336Z", "solve_latency_s": 150.00504884099792, "started_at": "2026-10-21T05:36:48.229322+00:00", "success": false, "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31660, "cache_read": 568345, "completion": 12598, "prompt": 604401, "reasoning": 0}, "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6643505000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "finished_at": "2026-10-21T05:41:26.636971+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.01504566499716, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1351__opus-5__a1__20261021T053716459589Z", "solve_latency_s": 250.01504566499716, "started_at": "2026-10-21T05:37:16.459575+00:00", "success": false, "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22166, "cache_read": 178506, "completion": 17456, "prompt": 200704, "reasoning": 14645}, "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15295620000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "finished_at": "2026-10-21T05:43:36.769493+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00649416599845, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1351__haiku-4.5__a1__20261021T054126643074Z", "solve_latency_s": 130.00649416599845, "started_at": "2026-10-21T05:41:26.643057+00:00", "success": false, "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22834, "cache_read": 589477, "completion": 9297, "prompt": 631292, "reasoning": 0}, "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5839949, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "finished_at": "2026-10-21T05:45:29.864353+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 350.0168433969993, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-980__sonnet-5__a1__20261021T053939818527Z", "solve_latency_s": 350.0168433969993, "started_at": "2026-10-21T05:39:39.818512+00:00", "success": false, "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41479, "cache_read": 1126637, "completion": 25477, "prompt": 1168216, "reasoning": 19409}, "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "latency_s": 38.35087448400009, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "latency_s": 41.11941400399883, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "latency_s": 41.171714132997295, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15427695, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.33930741199947, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "finished_at": "2026-10-21T05:50:58.905430+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.007131442002, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 42, "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1694__haiku-4.5__a1__20261021T054830533500Z", "solve_latency_s": 130.007131442002, "started_at": "2026-10-21T05:48:30.533484+00:00", "success": true, "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27979, "cache_read": 560672, "completion": 10121, "prompt": 601282, "reasoning": 0}, "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.055964200000000006, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.61069329699967, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "finished_at": "2026-10-21T05:52:08.575085+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.003515082000376, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 14, "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1694__sonnet-5__a1__20261021T055058916381Z", "solve_latency_s": 50.003515082000376, "started_at": "2026-10-21T05:50:58.916364+00:00", "success": true, "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 7204, "cache_read": 64641, "completion": 2497, "prompt": 71873, "reasoning": 133}, "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6562202500000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "finished_at": "2026-10-21T05:52:30.694706+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.00966935900215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 18, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "sqlfluff__sqlfluff-1763__opus-5__a1__20261021T054830536054Z", "solve_latency_s": 240.00966935900215, "started_at": "2026-10-21T05:48:30.536040+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18019, "cache_read": 205993, "completion": 17617, "prompt": 224048, "reasoning": 14798}, "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1998835, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.363647791000403, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "finished_at": "2026-10-21T05:53:27.098360+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.00267396199706, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 13, "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1694__opus-5__a1__20261021T055208585319Z", "solve_latency_s": 60.00267396199706, "started_at": "2026-10-21T05:52:08.585297+00:00", "success": true, "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10794, "cache_read": 83132, "completion": 3629, "prompt": 93952, "reasoning": 723}, "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5824565, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "finished_at": "2026-10-21T05:54:30.704714+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 360.0202696359993, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1089__sonnet-5__a1__20261021T054830530847Z", "solve_latency_s": 360.0202696359993, "started_at": "2026-10-21T05:48:30.530826+00:00", "success": false, "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 54023, "cache_read": 1456245, "completion": 15595, "prompt": 1510368, "reasoning": 6581}, "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15743615, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "finished_at": "2026-10-21T05:54:40.721426+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00434070999836, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "sqlfluff__sqlfluff-1763__haiku-4.5__a1__20261021T055230700767Z", "solve_latency_s": 130.00434070999836, "started_at": "2026-10-21T05:52:30.700753+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28741, "cache_read": 712399, "completion": 9265, "prompt": 745085, "reasoning": 0}, "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3926264, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "finished_at": "2026-10-21T05:58:30.932070+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.0084464870015, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "sqlfluff__sqlfluff-1763__sonnet-5__a1__20261021T055440728145Z", "solve_latency_s": 230.0084464870015, "started_at": "2026-10-21T05:54:40.728130+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31448, "cache_read": 834482, "completion": 14691, "prompt": 866030, "reasoning": 8396}, "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.8257777500000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "finished_at": "2026-10-21T06:00:40.855889+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 370.00920955400215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 10, "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1089__opus-5__a1__20261021T055430712199Z", "solve_latency_s": 370.00920955400215, "started_at": "2026-10-21T05:54:30.712185+00:00", "success": false, "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 9269, "cache_read": 58543, "completion": 29539, "prompt": 67832, "reasoning": 26974}, "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3122697, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "finished_at": "2026-10-21T06:05:01.032287+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 260.0080819939976, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1089__haiku-4.5__a1__20261021T060040862505Z", "solve_latency_s": 260.0080819939976, "started_at": "2026-10-21T06:00:40.862491+00:00", "success": false, "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 49426, "cache_read": 1280142, "completion": 21923, "prompt": 1342426, "reasoning": 0}, "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "latency_s": 40.129843676000746, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "latency_s": 36.54763845599882, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "latency_s": 43.66805849400407, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1904806, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.625123771998915, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "finished_at": "2026-10-21T06:09:42.763395+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00609180099855, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 24, "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1256__sonnet-5__a1__20261021T060743994333Z", "solve_latency_s": 100.00609180099855, "started_at": "2026-10-21T06:07:43.994319+00:00", "success": true, "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27276, "cache_read": 330273, "completion": 5614, "prompt": 357597, "reasoning": 801}, "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.22930945, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "finished_at": "2026-10-21T06:10:34.156342+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.0085420150026, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-2309__haiku-4.5__a1__20261021T060743991329Z", "solve_latency_s": 170.0085420150026, "started_at": "2026-10-21T06:07:43.991309+00:00", "success": false, "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 38463, "cache_read": 1030717, "completion": 13837, "prompt": 1078154, "reasoning": 0}, "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.14185150000000002, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.839329249000002, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "finished_at": "2026-10-21T06:10:51.796893+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.00240787599978, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 9, "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1256__opus-5__a1__20261021T060942770751Z", "solve_latency_s": 50.00240787599978, "started_at": "2026-10-21T06:09:42.770737+00:00", "success": true, "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8900, "cache_read": 54373, "completion": 2358, "prompt": 63291, "reasoning": 1029}, "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6296935, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "finished_at": "2026-10-21T06:11:34.139994+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00854100300057, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1666__opus-5__a1__20261021T060743996955Z", "solve_latency_s": 230.00854100300057, "started_at": "2026-10-21T06:07:43.996930+00:00", "success": false, "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23854, "cache_read": 129012, "completion": 16640, "prompt": 152886, "reasoning": 15203}, "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2276572, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.58577071900072, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "finished_at": "2026-10-21T06:14:00.596036+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.00615707200268, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 42, "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1256__haiku-4.5__a1__20261021T061051812807Z", "solve_latency_s": 170.00615707200268, "started_at": "2026-10-21T06:10:51.812793+00:00", "success": true, "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41972, "cache_read": 844972, "completion": 15251, "prompt": 901384, "reasoning": 0}, "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.4112056, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "finished_at": "2026-10-21T06:14:44.315089+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.00484274200062, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-2309__sonnet-5__a1__20261021T061034163941Z", "solve_latency_s": 250.00484274200062, "started_at": "2026-10-21T06:10:34.163926+00:00", "success": false, "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27984, "cache_read": 840728, "completion": 17290, "prompt": 868812, "reasoning": 10370}, "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.30990675, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "finished_at": "2026-10-21T06:16:14.303151+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.0077551350005, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1666__haiku-4.5__a1__20261021T061134148249Z", "solve_latency_s": 280.0077551350005, "started_at": "2026-10-21T06:11:34.148235+00:00", "success": false, "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 51087, "cache_read": 1099000, "completion": 26248, "prompt": 1154995, "reasoning": 0}, "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.64450475, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "finished_at": "2026-10-21T06:18:34.504243+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00638435900328, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 29, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-2309__opus-5__a1__20261021T061444322645Z", "solve_latency_s": 230.00638435900328, "started_at": "2026-10-21T06:14:44.322630+00:00", "success": false, "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22259, "cache_read": 413692, "completion": 11930, "prompt": 436009, "reasoning": 7127}, "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"} +{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5550333, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.54369581699939, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "finished_at": "2026-10-21T06:21:26.049958+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 290.00907700800235, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1666__sonnet-5__a1__20261021T061614312013Z", "solve_latency_s": 290.00907700800235, "started_at": "2026-10-21T06:16:14.311997+00:00", "success": false, "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 46207, "cache_read": 1151729, "completion": 20897, "prompt": 1198036, "reasoning": 11581}, "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "latency_s": 42.26520768800037, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "latency_s": 41.60962831600045, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1365", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "latency_s": 37.42055031199925, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.12878715, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.432110554000246, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "finished_at": "2026-10-21T06:26:14.194867+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00872880399766, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 45, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1978__haiku-4.5__a1__20261021T062352591716Z", "solve_latency_s": 120.00872880399766, "started_at": "2026-10-21T06:23:52.591693+00:00", "success": false, "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18261, "cache_read": 377229, "completion": 7221, "prompt": 427623, "reasoning": 0}, "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.039051100000000005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.536053373001778, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "finished_at": "2026-10-21T06:27:15.945861+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.002291953001986, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 12, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1978__sonnet-5__a1__20261021T062614202893Z", "solve_latency_s": 40.002291953001986, "started_at": "2026-10-21T06:26:14.202878+00:00", "success": false, "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 5999, "cache_read": 47228, "completion": 1456, "prompt": 53251, "reasoning": 157}, "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.76345675, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "finished_at": "2026-10-21T06:28:32.730831+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.00734789200214, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-933__opus-5__a1__20261021T062352595828Z", "solve_latency_s": 280.00734789200214, "started_at": "2026-10-21T06:23:52.595808+00:00", "success": false, "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17933, "cache_read": 91051, "completion": 24230, "prompt": 109004, "reasoning": 21276}, "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1721697, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "finished_at": "2026-10-21T06:30:42.878170+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00383345900264, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-933__haiku-4.5__a1__20261021T062832739377Z", "solve_latency_s": 130.00383345900264, "started_at": "2026-10-21T06:28:32.739361+00:00", "success": false, "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28608, "cache_read": 706207, "completion": 9942, "prompt": 750894, "reasoning": 0}, "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.68405275, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "finished_at": "2026-10-21T06:31:46.089536+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 270.0068097680014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 15, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1978__opus-5__a1__20261021T062715954897Z", "solve_latency_s": 270.0068097680014, "started_at": "2026-10-21T06:27:15.954882+00:00", "success": false, "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 14751, "cache_read": 128468, "completion": 21099, "prompt": 143249, "reasoning": 17506}, "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2785342, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.521670827998605, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "finished_at": "2026-10-21T06:33:31.557766+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00452539499747, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 30, "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-933__sonnet-5__a1__20261021T063042889310Z", "solve_latency_s": 150.00452539499747, "started_at": "2026-10-21T06:30:42.889294+00:00", "success": true, "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27446, "cache_read": 459296, "completion": 11794, "prompt": 486802, "reasoning": 5417}, "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "latency_s": 53.46030479900219, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4414", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"} +{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "latency_s": 39.439905870000075, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1048", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "latency_s": 36.34352293600023, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.633041, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "finished_at": "2026-10-21T06:39:40.710035+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0060526820016, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 22, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1333__opus-5__a1__20261021T063630543785Z", "solve_latency_s": 190.0060526820016, "started_at": "2026-10-21T06:36:30.543760+00:00", "success": false, "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24306, "cache_read": 309167, "completion": 13053, "prompt": 333517, "reasoning": 9773}, "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.18808655, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "finished_at": "2026-10-21T06:42:20.906468+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.004997172, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1333__haiku-4.5__a1__20261021T063940718628Z", "solve_latency_s": 160.004997172, "started_at": "2026-10-21T06:39:40.718612+00:00", "success": false, "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 32485, "cache_read": 732533, "completion": 13084, "prompt": 773825, "reasoning": 0}, "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"} +{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.07471950000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.579913778001355, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "finished_at": "2026-10-21T06:43:39.651573+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.00300859399795, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 18, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1333__sonnet-5__a1__20261021T064220926048Z", "solve_latency_s": 60.00300859399795, "started_at": "2026-10-21T06:42:20.926031+00:00", "success": true, "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10423, "cache_read": 115900, "completion": 2541, "prompt": 126359, "reasoning": 495}, "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"}