scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

chore(capability_routing_pilot): add final profiling checkpoint and capability map

Browse source 
Co-Authored-By: bot_apk <apk@cognition.ai>
This commit is contained in:
Devin AI 2026-10-21 12:30:17 +00:00
parent 4c2abbc905
commit 9b68bb9833
2 changed files with 1092 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

802
cookbook/capability_routing_pilot/profiling/checkpoints/final/capability_map_final.json Normal file
View file

@ -0,0 +1,802 @@
{
"accounting": {
"attempt_count": 171,
"full_attempt_spend_usd": 62.033954,
"graded_count": 171,
"infra_count": 0,
"successes": 47
},
"category_coverage": {
"algorithms_data": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"feature_implementation": {
"profile_count": 3,
"status_counts": {
"evidenced_below_target": 3,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"repository_debugging": {
"profile_count": 3,
"status_counts": {
"evidenced_below_target": 3,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"shell_build": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"unknown": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
}
},
"category_decisions": {
"algorithms_data": {
"decision": "unsupported_category",
"reason": "unsupported_category",
"selected_profile": null,
"threshold_unmet": false
},
"feature_implementation": {
"decision": "no_qualified_model",
"reason": "no_qualified_model",
"selected_profile": null,
"threshold_unmet": false
},
"repository_debugging": {
"decision": "no_qualified_model",
"reason": "no_qualified_model",
"selected_profile": null,
"threshold_unmet": false
},
"shell_build": {
"decision": "unsupported_category",
"reason": "unsupported_category",
"selected_profile": null,
"threshold_unmet": false
},
"unknown": {
"decision": "unsupported_category",
"reason": "unsupported_category",
"selected_profile": null,
"threshold_unmet": false
}
},
"compiler_final_flag": true,
"environment_deficits": [
"pyvista__pyvista-432"
],
"final": true,
"gateway_identity_note": "All three aliases share gateway model-info id 13ef3d99-7edf-4a6b-aab6-f088c8e25964, a gateway mapping snapshot, not model identity. Cohorts are separated by operational alias and returned model name",
"infra_deficits": [],
"interpretation": "Beta(1,1) credible bound describes the exchangeable public-dev task distribution for this category, not guaranteed per-task success",
"min_evidence_note": "necessary not sufficient: 13/13 is the minimum perfect cell clearing 0.80; not a general adequacy claim",
"pending_environment_repairs": [],
"preflight_failed": [
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest",
"latency_s": 36.985300702999666,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1413",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460",
"upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest",
"latency_s": 45.86221657400165,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1439",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517",
"upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest",
"latency_s": 59.88555217099929,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-4329",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3",
"upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest",
"latency_s": 37.34210575900215,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1720",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337",
"upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-995",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-995:latest",
"latency_s": 37.53449416300282,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-995",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39",
"upstream_image_id": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest",
"latency_s": 40.2318156500005,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1334",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d",
"upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest",
"latency_s": 41.60962831600045,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1365",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2",
"upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest",
"latency_s": 53.46030479900219,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-4414",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92",
"upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest",
"latency_s": 39.439905870000075,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1048",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f",
"upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest",
"category": "feature_implementation",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest",
"latency_s": 44.45388161699884,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-3675",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7",
"upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest",
"category": "feature_implementation",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest",
"latency_s": 38.152053487003286,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-432",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21",
"upstream_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428",
"category": "feature_implementation",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:68636bf54706b0f82a31980d5cb719e6056f8129a4d79f3d3a47f67f478f2e49",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest",
"latency_s": 49.36457281099865,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:68636bf54706b0f82a31980d5cb719e6056f8129a4d79f3d3a47f67f478f2e49",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1428",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e",
"upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest",
"category": "feature_implementation",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest",
"latency_s": 67.93332233199908,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-4648",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455",
"upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest",
"category": "feature_implementation",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest",
"latency_s": 44.1322110139954,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"opus-5",
"haiku-4.5",
"sonnet-5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-4311",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453",
"upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439-envfix1",
"applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)",
"category": "repository_debugging",
"cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; TestNumpy_NoRLEHandler and TestNumpy_RLEHandler define setup(self) to restrict pixel handlers, so both tests saw default handlers and failed",
"corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719",
"cost_usd": 0.0,
"dockerfile_sha256": "537a4abfca69b6009fc410550b85e90d103ace6be8e725e0e31e184fcb80c970",
"env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719",
"error_class": "env",
"evidence": [
"pydicom/tests/test_rle_pixel_data.py::TestNumpy_NoRLEHandler::test_pixel_array_raises",
"pydicom/tests/test_rle_pixel_data.py::TestNumpy_RLEHandler::test_unsupported_syntax_raises"
],
"final_image_id": "sha256:7523d0b5c2cf51e01a7d93089bc7a1dad29ee9baf4a857b45165d304c531228e",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest",
"latency_s": 44.88570009100658,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:7523d0b5c2cf51e01a7d93089bc7a1dad29ee9baf4a857b45165d304c531228e",
"record_type": "preflight_failed",
"repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1439",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517",
"upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-432-envfix1",
"applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)",
"category": "feature_implementation",
"cause": "Gold and noop preflights both failed at collection with ImportError: libXrender.so.1: cannot open shared object file, so no FAIL_TO_PASS test could run; the image lacks the X11 render library VTK loads at import",
"corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb",
"cost_usd": 0.0,
"dockerfile_sha256": "b62d6429e68a1e8f92a8d8c2cdbdedae5d0b43571d4eb724c5e094b5acda2c32",
"env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb",
"error_class": "env",
"evidence": [
"tests/test_plotting.py::test_invalid_color",
"tests/test_plotting.py::test_invalid_font"
],
"final_image_id": "sha256:4e818cdda198e7fac1aa95e7a29c9446f398d918b97ce5cc651ea571a141d0e9",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest",
"latency_s": 38.10395569400134,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:4e818cdda198e7fac1aa95e7a29c9446f398d918b97ce5cc651ea571a141d0e9",
"record_type": "preflight_failed",
"repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-432",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21",
"upstream_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21"
}
],
"preflight_incomplete": [],
"profiles": [
{
"attempt_count": 29,
"beta_mean": 0.0967741935483871,
"cost_per_graded_task_usd": 0.2329313827586207,
"graded_count": 29,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "feature_implementation",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "haiku-4.5",
"returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.02781554973974965,
"mean_latency_s": 179.66291863237925,
"min_evidence": 13,
"primary_labels": [
"feature_implementation"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "haiku-4.5"
},
"raw_rate": 0.06896551724137931,
"secondary_labels": [],
"status": "evidenced_below_target",
"successes": 2,
"tau": 0.8,
"total_spend_usd": 6.7550101
},
{
"attempt_count": 29,
"beta_mean": 0.25806451612903225,
"cost_per_graded_task_usd": 0.5569036551724138,
"graded_count": 29,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "feature_implementation",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "opus-5",
"returned_model_version": "litellm_proxy/anthropic/claude-opus-5",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.14018509946776775,
"mean_latency_s": 165.5237464495169,
"min_evidence": 13,
"primary_labels": [
"feature_implementation"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "opus-5"
},
"raw_rate": 0.2413793103448276,
"secondary_labels": [],
"status": "evidenced_below_target",
"successes": 7,
"tau": 0.8,
"total_spend_usd": 16.150206
},
{
"attempt_count": 29,
"beta_mean": 0.3548387096774194,
"cost_per_graded_task_usd": 0.3745770931034483,
"graded_count": 29,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "feature_implementation",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "sonnet-5",
"returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.22105944984601347,
"mean_latency_s": 165.86940478075798,
"min_evidence": 13,
"primary_labels": [
"feature_implementation"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "sonnet-5"
},
"raw_rate": 0.3448275862068966,
"secondary_labels": [],
"status": "evidenced_below_target",
"successes": 10,
"tau": 0.8,
"total_spend_usd": 10.8627357
},
{
"attempt_count": 28,
"beta_mean": 0.26666666666666666,
"cost_per_graded_task_usd": 0.18977413214285715,
"graded_count": 28,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "repository_debugging",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "haiku-4.5",
"returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.1453215823182127,
"mean_latency_s": 152.14831222699974,
"min_evidence": 13,
"primary_labels": [
"repository_debugging"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "haiku-4.5"
},
"raw_rate": 0.25,
"secondary_labels": [],
"status": "evidenced_below_target",
"successes": 7,
"tau": 0.8,
"total_spend_usd": 5.3136757
},
{
"attempt_count": 28,
"beta_mean": 0.3,
"cost_per_graded_task_usd": 0.5271966071428571,
"graded_count": 28,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "repository_debugging",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "opus-5",
"returned_model_version": "litellm_proxy/anthropic/claude-opus-5",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.17246481940361086,
"mean_latency_s": 177.86307842582195,
"min_evidence": 13,
"primary_labels": [
"repository_debugging"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "opus-5"
},
"raw_rate": 0.2857142857142857,
"secondary_labels": [],
"status": "evidenced_below_target",
"successes": 8,
"tau": 0.8,
"total_spend_usd": 14.761505
},
{
"attempt_count": 28,
"beta_mean": 0.4666666666666667,
"cost_per_graded_task_usd": 0.29252935357142856,
"graded_count": 28,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "repository_debugging",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "sonnet-5",
"returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.32029626073802414,
"mean_latency_s": 171.79238967996415,
"min_evidence": 13,
"primary_labels": [
"repository_debugging"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "sonnet-5"
},
"raw_rate": 0.4642857142857143,
"secondary_labels": [],
"status": "evidenced_below_target",
"successes": 13,
"tau": 0.8,
"total_spend_usd": 8.1908219
}
],
"quarantined_attempts": [],
"repaired_tasks": [
"pydicom__pydicom-1048",
"pydicom__pydicom-1334",
"pydicom__pydicom-1365",
"pydicom__pydicom-1413",
"pydicom__pydicom-1428",
"pydicom__pydicom-1720",
"pyvista__pyvista-3675",
"pyvista__pyvista-4311",
"pyvista__pyvista-4329",
"pyvista__pyvista-4414",
"pyvista__pyvista-4648"
],
"spec_evaluator_deficits": [
{
"reason": "20 PASS_TO_PASS IDs embed original-machine paths, corrected gold passes 80 with 0 failed",
"task_id": "pydicom__pydicom-1439"
},
{
"reason": "44 FAIL_TO_PASS IDs embed original-machine temp paths, gold passes 256/256 but cannot match",
"task_id": "pydicom__pydicom-995"
}
],
"tenant_id": "public_dev_poc",
"thresholds": {
"min_evidence": 13,
"tau": 0.8
},
"versions": {
"budget_version": "budget_1usd_50_600_v1",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_deployments": {
"haiku-4.5": [
"13ef3d99-7edf-4a6b-aab6-f088c8e25964"
],
"opus-5": [
"13ef3d99-7edf-4a6b-aab6-f088c8e25964"
],
"sonnet-5": [
"13ef3d99-7edf-4a6b-aab6-f088c8e25964"
]
},
"model_versions": {
"haiku-4.5": [
"litellm_proxy/anthropic/claude-haiku-4-5-20251001"
],
"opus-5": [
"litellm_proxy/anthropic/claude-opus-5"
],
"sonnet-5": [
"litellm_proxy/anthropic/claude-sonnet-5"
]
},
"observed_model_cohorts": [
{
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"model_alias": "haiku-4.5",
"returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001"
},
{
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"model_alias": "opus-5",
"returned_model_version": "litellm_proxy/anthropic/claude-opus-5"
},
{
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"model_alias": "sonnet-5",
"returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5"
}
],
"version_changes": []
}
}

290
cookbook/capability_routing_pilot/profiling/checkpoints/final/task_runs.jsonl Normal file
View file

@ -0,0 +1,290 @@
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "latency_s": 36.985300702999666, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "latency_s": 37.443159876998834, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "latency_s": 42.63817018199916, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15246375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.22013637100099, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:29:42.514070+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.00210723499913, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 11, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1216__opus-5__a1__20261021T042841152530Z", "solve_latency_s": 40.00210723499913, "started_at": "2026-10-21T04:28:41.152510+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11155, "cache_read": 72220, "completion": 1861, "prompt": 83397, "reasoning": 34}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0754433, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.261532576998434, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:29:49.569191+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.00336846000209, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 15, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1192__sonnet-5__a1__20261021T042841151566Z", "solve_latency_s": 50.00336846000209, "started_at": "2026-10-21T04:28:41.151547+00:00", "success": true, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11487, "cache_read": 101329, "completion": 2640, "prompt": 112846, "reasoning": 66}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.08664235, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.42284784900039, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:31:34.083967+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00328306499796, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 25, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1216__haiku-4.5__a1__20261021T042942515886Z", "solve_latency_s": 90.00328306499796, "started_at": "2026-10-21T04:29:42.515874+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17291, "cache_read": 192516, "completion": 6628, "prompt": 222444, "reasoning": 0}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.38297225, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 20.581349566000426, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:32:00.355179+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.00375107299988, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 20, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1192__opus-5__a1__20261021T042949571022Z", "solve_latency_s": 110.00375107299988, "started_at": "2026-10-21T04:29:49.571011+00:00", "success": true, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17105, "cache_read": 226732, "completion": 6500, "prompt": 243877, "reasoning": 3366}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0498986, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.37230331899991, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:32:35.584861+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.00220326099952, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1216__sonnet-5__a1__20261021T043134086373Z", "solve_latency_s": 40.00220326099952, "started_at": "2026-10-21T04:31:34.086359+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8744, "cache_read": 53193, "completion": 1736, "prompt": 61957, "reasoning": 10}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2071005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:34:50.522036+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.0046138639991, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1192__haiku-4.5__a1__20261021T043200357212Z", "solve_latency_s": 170.0046138639991, "started_at": "2026-10-21T04:32:00.357202+00:00", "success": false, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31386, "cache_read": 742450, "completion": 15459, "prompt": 790164, "reasoning": 0}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest", "latency_s": 45.86221657400165, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1439", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517", "upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "latency_s": 46.50070145599966, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "latency_s": 39.32556367899815, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6126465, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:47:21.107529+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00617045400213, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 27, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-2307__opus-5__a1__20261021T044400932430Z", "solve_latency_s": 200.00617045400213, "started_at": "2026-10-21T04:44:00.932414+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23326, "cache_read": 365428, "completion": 11355, "prompt": 388808, "reasoning": 6719}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3954644, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 22.276458722000825, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:48:33.342443+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.0124097070002, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 32, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1165__sonnet-5__a1__20261021T044400931150Z", "solve_latency_s": 250.0124097070002, "started_at": "2026-10-21T04:44:00.931128+00:00", "success": true, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30230, "cache_read": 513457, "completion": 21707, "prompt": 543751, "reasoning": 15209}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1836434, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:50:01.255188+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00828153600014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-2307__haiku-4.5__a1__20261021T044721112372Z", "solve_latency_s": 160.00828153600014, "started_at": "2026-10-21T04:47:21.112358+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 29088, "cache_read": 697624, "completion": 12455, "prompt": 741958, "reasoning": 0}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.626164, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:51:33.536769+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00563653500285, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 13, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1165__opus-5__a1__20261021T044833345077Z", "solve_latency_s": 180.00563653500285, "started_at": "2026-10-21T04:48:33.345065+00:00", "success": false, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24668, "cache_read": 184418, "completion": 15186, "prompt": 209112, "reasoning": 12703}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.26214350000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:55:23.672687+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00504096200166, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1165__haiku-4.5__a1__20261021T045133539818Z", "solve_latency_s": 230.00504096200166, "started_at": "2026-10-21T04:51:33.539806+00:00", "success": false, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 43394, "cache_read": 976470, "completion": 19432, "prompt": 1032958, "reasoning": 0}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5375916000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:55:41.469860+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 340.0079844879983, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-2307__sonnet-5__a1__20261021T045001257820Z", "solve_latency_s": 340.0079844879983, "started_at": "2026-10-21T04:50:01.257807+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37278, "cache_read": 973333, "completion": 24953, "prompt": 1010711, "reasoning": 15582}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "latency_s": 59.88555217099929, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "latency_s": 37.34210575900215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "latency_s": 36.9401233719982, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.49552525000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.36470413599818, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:01:43.053923+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00416897800096, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 19, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-897__opus-5__a1__20261021T045854563009Z", "solve_latency_s": 150.00416897800096, "started_at": "2026-10-21T04:58:54.562993+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19647, "cache_read": 236183, "completion": 10178, "prompt": 255868, "reasoning": 5998}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15681435, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:03:53.181859+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.0038786289988, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-897__haiku-4.5__a1__20261021T050143057007Z", "solve_latency_s": 130.0038786289988, "started_at": "2026-10-21T05:01:43.056996+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23533, "cache_read": 670341, "completion": 9320, "prompt": 707638, "reasoning": 0}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2344321, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.46944500400059, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:06:41.825678+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00388437999936, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 45, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-897__sonnet-5__a1__20261021T050353184991Z", "solve_latency_s": 150.00388437999936, "started_at": "2026-10-21T05:03:53.184978+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18927, "cache_read": 459623, "completion": 9501, "prompt": 478640, "reasoning": 3384}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "latency_s": 39.889730383001734, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-995", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-995:latest", "latency_s": 37.53449416300282, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-995", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39", "upstream_image_id": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "latency_s": 39.2809497240014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.32255725, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.878957515000366, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:11:02.894305+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 80.00347018200046, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 13, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1349__opus-5__a1__20261021T050922867062Z", "solve_latency_s": 80.00347018200046, "started_at": "2026-10-21T05:09:22.867049+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18737, "cache_read": 157992, "completion": 5053, "prompt": 176755, "reasoning": 2608}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1955686, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:11:23.004924+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00499666500036, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1364__haiku-4.5__a1__20261021T050922865588Z", "solve_latency_s": 120.00499666500036, "started_at": "2026-10-21T05:09:22.865569+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37378, "cache_read": 1011631, "completion": 7784, "prompt": 1057772, "reasoning": 0}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.25508825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.928946245003317, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:14:22.849769+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00484361299823, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 46, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1349__haiku-4.5__a1__20261021T051102898250Z", "solve_latency_s": 180.00484361299823, "started_at": "2026-10-21T05:11:02.898236+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 42975, "cache_read": 1255085, "completion": 13855, "prompt": 1304646, "reasoning": 0}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1126159, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.824896968002577, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:15:32.696323+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.0022958030022, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 14, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1349__sonnet-5__a1__20261021T051422853521Z", "solve_latency_s": 50.0022958030022, "started_at": "2026-10-21T05:14:22.853508+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18231, "cache_read": 170812, "completion": 3282, "prompt": 189071, "reasoning": 872}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.4375135, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:16:03.478380+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.0084194079973, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1364__sonnet-5__a1__20261021T051123009015Z", "solve_latency_s": 280.0084194079973, "started_at": "2026-10-21T05:11:23.009002+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35875, "cache_read": 974680, "completion": 15269, "prompt": 1010655, "reasoning": 8390}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.648212, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:20:13.628921+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.0070123559999, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1364__opus-5__a1__20261021T051603482193Z", "solve_latency_s": 250.0070123559999, "started_at": "2026-10-21T05:16:03.482180+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19296, "cache_read": 169404, "completion": 17710, "prompt": 188732, "reasoning": 15644}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "latency_s": 39.77621123099743, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "latency_s": 43.17733980900084, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "latency_s": 37.75819920799768, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0995685, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.37739269600206, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:23:28.637543+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.001993197998672, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 6, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1164__opus-5__a1__20261021T052240132919Z", "solve_latency_s": 30.001993197998672, "started_at": "2026-10-21T05:22:40.132907+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 7776, "cache_read": 27567, "completion": 1485, "prompt": 35355, "reasoning": 253}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.08022180000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.130332180997357, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:23:51.415788+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.007217943002615, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1719__sonnet-5__a1__20261021T052240127823Z", "solve_latency_s": 50.007217943002615, "started_at": "2026-10-21T05:22:40.127805+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11136, "cache_read": 111339, "completion": 3005, "prompt": 122507, "reasoning": 523}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.18115045, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.322718805000477, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:25:09.592869+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00579304400162, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 46, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1810__haiku-4.5__a1__20261021T052240130927Z", "solve_latency_s": 130.00579304400162, "started_at": "2026-10-21T05:22:40.130915+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33073, "cache_read": 697242, "completion": 10692, "prompt": 746940, "reasoning": 0}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.12290365, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.234139322001283, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:25:27.046567+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00316618500074, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 39, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1164__haiku-4.5__a1__20261021T052328641925Z", "solve_latency_s": 100.00316618500074, "started_at": "2026-10-21T05:23:28.641910+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 21985, "cache_read": 420494, "completion": 7907, "prompt": 456317, "reasoning": 0}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.035914600000000005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.89811670599738, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:25:59.649361+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.00381014999948, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 9, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1810__sonnet-5__a1__20261021T052509597736Z", "solve_latency_s": 30.00381014999948, "started_at": "2026-10-21T05:25:09.597723+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 6304, "cache_read": 35943, "completion": 1293, "prompt": 42265, "reasoning": 40}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.031602, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.185294439001154, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:26:16.378423+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.00225875400065, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 8, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1164__sonnet-5__a1__20261021T052527051475Z", "solve_latency_s": 30.00225875400065, "started_at": "2026-10-21T05:25:27.051461+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 5698, "cache_read": 27125, "completion": 1190, "prompt": 32839, "reasoning": 61}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.23039375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.577401711001585, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:27:49.390345+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00354062100087, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 8, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1810__opus-5__a1__20261021T052559657755Z", "solve_latency_s": 90.00354062100087, "started_at": "2026-10-21T05:25:59.657740+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8455, "cache_read": 41590, "completion": 6267, "prompt": 50061, "reasoning": 4667}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.66210825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 23.277767100000347, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:28:14.846207+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.00587134300076, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 27, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1719__opus-5__a1__20261021T052351420018Z", "solve_latency_s": 240.00587134300076, "started_at": "2026-10-21T05:23:51.420006+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 20941, "cache_read": 368864, "completion": 13861, "prompt": 389859, "reasoning": 8383}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1483624, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.447099036999134, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:31:06.320651+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00421196300158, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 42, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1719__haiku-4.5__a1__20261021T052814852065Z", "solve_latency_s": 150.00421196300158, "started_at": "2026-10-21T05:28:14.852050+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25984, "cache_read": 629064, "completion": 9321, "prompt": 661419, "reasoning": 0}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "latency_s": 40.2318156500005, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:daaf2bfb94a9d94be15642ee0037fa7862ffe8c9a22519615679db63bf5ae2da", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1334", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "latency_s": 47.930545192000864, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "latency_s": 43.92638719699971, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.46326425000000004, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.742469348002487, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "finished_at": "2026-10-21T05:36:48.224174+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.0067291660016, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 16, "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-980__opus-5__a1__20261021T053406317144Z", "solve_latency_s": 140.0067291660016, "started_at": "2026-10-21T05:34:06.317128+00:00", "success": false, "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22519, "cache_read": 216521, "completion": 8564, "prompt": 239072, "reasoning": 5572}, "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.30063480000000004, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "finished_at": "2026-10-21T05:37:16.454317+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0117116170004, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1351__sonnet-5__a1__20261021T053406312824Z", "solve_latency_s": 190.0117116170004, "started_at": "2026-10-21T05:34:06.312804+00:00", "success": false, "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19556, "cache_read": 538174, "completion": 14391, "prompt": 557830, "reasoning": 8757}, "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1637955, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.43748659799894, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "finished_at": "2026-10-21T05:39:39.812688+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00504884099792, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 34, "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-980__haiku-4.5__a1__20261021T053648229336Z", "solve_latency_s": 150.00504884099792, "started_at": "2026-10-21T05:36:48.229322+00:00", "success": false, "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31660, "cache_read": 568345, "completion": 12598, "prompt": 604401, "reasoning": 0}, "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6643505000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "finished_at": "2026-10-21T05:41:26.636971+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.01504566499716, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1351__opus-5__a1__20261021T053716459589Z", "solve_latency_s": 250.01504566499716, "started_at": "2026-10-21T05:37:16.459575+00:00", "success": false, "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22166, "cache_read": 178506, "completion": 17456, "prompt": 200704, "reasoning": 14645}, "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15295620000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "finished_at": "2026-10-21T05:43:36.769493+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1351:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00649416599845, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1351__haiku-4.5__a1__20261021T054126643074Z", "solve_latency_s": 130.00649416599845, "started_at": "2026-10-21T05:41:26.643057+00:00", "success": false, "task_id": "pylint-dev__astroid-1351", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22834, "cache_read": 589477, "completion": 9297, "prompt": 631292, "reasoning": 0}, "upstream_digest": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9", "upstream_image_id": "sha256:1b6085986aef88934ea55ef44e643b83b4ebb603c1c1d810fd06f695eaf1aec9"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-980", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5839949, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "finished_at": "2026-10-21T05:45:29.864353+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-980:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 350.0168433969993, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:5ca8e2674fd824812f6f8f9f58acc7c99245ac7eee4725ccd3961a8efe127dc3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-980__sonnet-5__a1__20261021T053939818527Z", "solve_latency_s": 350.0168433969993, "started_at": "2026-10-21T05:39:39.818512+00:00", "success": false, "task_id": "pvlib__pvlib-python-980", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41479, "cache_read": 1126637, "completion": 25477, "prompt": 1168216, "reasoning": 19409}, "upstream_digest": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5", "upstream_image_id": "sha256:d6b63248f54f4b5dc4d01eea778980095adf6d797ff78d56a7610f5dfab3dfe5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "latency_s": 38.35087448400009, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "latency_s": 41.11941400399883, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "latency_s": 41.171714132997295, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15427695, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.33930741199947, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "finished_at": "2026-10-21T05:50:58.905430+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.007131442002, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 42, "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1694__haiku-4.5__a1__20261021T054830533500Z", "solve_latency_s": 130.007131442002, "started_at": "2026-10-21T05:48:30.533484+00:00", "success": true, "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27979, "cache_read": 560672, "completion": 10121, "prompt": 601282, "reasoning": 0}, "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.055964200000000006, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.61069329699967, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "finished_at": "2026-10-21T05:52:08.575085+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.003515082000376, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 14, "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1694__sonnet-5__a1__20261021T055058916381Z", "solve_latency_s": 50.003515082000376, "started_at": "2026-10-21T05:50:58.916364+00:00", "success": true, "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 7204, "cache_read": 64641, "completion": 2497, "prompt": 71873, "reasoning": 133}, "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6562202500000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "finished_at": "2026-10-21T05:52:30.694706+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.00966935900215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 18, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "sqlfluff__sqlfluff-1763__opus-5__a1__20261021T054830536054Z", "solve_latency_s": 240.00966935900215, "started_at": "2026-10-21T05:48:30.536040+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18019, "cache_read": 205993, "completion": 17617, "prompt": 224048, "reasoning": 14798}, "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1694", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1998835, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.363647791000403, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "finished_at": "2026-10-21T05:53:27.098360+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1694:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.00267396199706, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 13, "overlay_image_id": "sha256:5291dfbb08388cd406c342b7593933691c25b9c60cfb468a4bd62c2366d2ccf7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1694__opus-5__a1__20261021T055208585319Z", "solve_latency_s": 60.00267396199706, "started_at": "2026-10-21T05:52:08.585297+00:00", "success": true, "task_id": "pydicom__pydicom-1694", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10794, "cache_read": 83132, "completion": 3629, "prompt": 93952, "reasoning": 723}, "upstream_digest": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12", "upstream_image_id": "sha256:80521627a8f3f69682f16743158b1bc8555c9253c42164a4b59050293d84ba12"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5824565, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "finished_at": "2026-10-21T05:54:30.704714+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 360.0202696359993, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1089__sonnet-5__a1__20261021T054830530847Z", "solve_latency_s": 360.0202696359993, "started_at": "2026-10-21T05:48:30.530826+00:00", "success": false, "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 54023, "cache_read": 1456245, "completion": 15595, "prompt": 1510368, "reasoning": 6581}, "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15743615, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "finished_at": "2026-10-21T05:54:40.721426+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00434070999836, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "sqlfluff__sqlfluff-1763__haiku-4.5__a1__20261021T055230700767Z", "solve_latency_s": 130.00434070999836, "started_at": "2026-10-21T05:52:30.700753+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28741, "cache_read": 712399, "completion": 9265, "prompt": 745085, "reasoning": 0}, "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3926264, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "finished_at": "2026-10-21T05:58:30.932070+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-1763:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.0084464870015, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "sqlfluff__sqlfluff-1763__sonnet-5__a1__20261021T055440728145Z", "solve_latency_s": 230.0084464870015, "started_at": "2026-10-21T05:54:40.728130+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-1763", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31448, "cache_read": 834482, "completion": 14691, "prompt": 866030, "reasoning": 8396}, "upstream_digest": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b", "upstream_image_id": "sha256:6fed0c9e91988d87b75b0f77fcc6fcdcb4a35ecfc5ba20c902b0656d5855570b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.8257777500000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "finished_at": "2026-10-21T06:00:40.855889+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 370.00920955400215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 10, "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1089__opus-5__a1__20261021T055430712199Z", "solve_latency_s": 370.00920955400215, "started_at": "2026-10-21T05:54:30.712185+00:00", "success": false, "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 9269, "cache_read": 58543, "completion": 29539, "prompt": 67832, "reasoning": 26974}, "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1089", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3122697, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "finished_at": "2026-10-21T06:05:01.032287+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1089:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 260.0080819939976, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:a858392c016db3feda53375bcb514ab99c02399a43e4eb8262ca00df0c17b3db", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1089__haiku-4.5__a1__20261021T060040862505Z", "solve_latency_s": 260.0080819939976, "started_at": "2026-10-21T06:00:40.862491+00:00", "success": false, "task_id": "pvlib__pvlib-python-1089", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 49426, "cache_read": 1280142, "completion": 21923, "prompt": 1342426, "reasoning": 0}, "upstream_digest": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d", "upstream_image_id": "sha256:a7168b4effc48480c2ce20af505337a0a0a7cd19fda616784af104a9c40d912d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "latency_s": 40.129843676000746, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "latency_s": 36.54763845599882, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "latency_s": 43.66805849400407, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1904806, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.625123771998915, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "finished_at": "2026-10-21T06:09:42.763395+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00609180099855, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 24, "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1256__sonnet-5__a1__20261021T060743994333Z", "solve_latency_s": 100.00609180099855, "started_at": "2026-10-21T06:07:43.994319+00:00", "success": true, "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27276, "cache_read": 330273, "completion": 5614, "prompt": 357597, "reasoning": 801}, "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.22930945, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "finished_at": "2026-10-21T06:10:34.156342+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.0085420150026, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-2309__haiku-4.5__a1__20261021T060743991329Z", "solve_latency_s": 170.0085420150026, "started_at": "2026-10-21T06:07:43.991309+00:00", "success": false, "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 38463, "cache_read": 1030717, "completion": 13837, "prompt": 1078154, "reasoning": 0}, "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.14185150000000002, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.839329249000002, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "finished_at": "2026-10-21T06:10:51.796893+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.00240787599978, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 9, "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1256__opus-5__a1__20261021T060942770751Z", "solve_latency_s": 50.00240787599978, "started_at": "2026-10-21T06:09:42.770737+00:00", "success": true, "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8900, "cache_read": 54373, "completion": 2358, "prompt": 63291, "reasoning": 1029}, "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6296935, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "finished_at": "2026-10-21T06:11:34.139994+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00854100300057, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1666__opus-5__a1__20261021T060743996955Z", "solve_latency_s": 230.00854100300057, "started_at": "2026-10-21T06:07:43.996930+00:00", "success": false, "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23854, "cache_read": 129012, "completion": 16640, "prompt": 152886, "reasoning": 15203}, "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1256", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2276572, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.58577071900072, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "finished_at": "2026-10-21T06:14:00.596036+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1256:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.00615707200268, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 42, "overlay_image_id": "sha256:6c5daa247f9790a9f2850d8f8a063fb4e79644a6b4e0735c9cd4a7c26889f450", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1256__haiku-4.5__a1__20261021T061051812807Z", "solve_latency_s": 170.00615707200268, "started_at": "2026-10-21T06:10:51.812793+00:00", "success": true, "task_id": "pydicom__pydicom-1256", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41972, "cache_read": 844972, "completion": 15251, "prompt": 901384, "reasoning": 0}, "upstream_digest": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd", "upstream_image_id": "sha256:6443432fb247833fa2a2158e751ef62817e36d0a0219aa89ada44bfe7129a3fd"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.4112056, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "finished_at": "2026-10-21T06:14:44.315089+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.00484274200062, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-2309__sonnet-5__a1__20261021T061034163941Z", "solve_latency_s": 250.00484274200062, "started_at": "2026-10-21T06:10:34.163926+00:00", "success": false, "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27984, "cache_read": 840728, "completion": 17290, "prompt": 868812, "reasoning": 10370}, "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.30990675, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "finished_at": "2026-10-21T06:16:14.303151+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.0077551350005, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1666__haiku-4.5__a1__20261021T061134148249Z", "solve_latency_s": 280.0077551350005, "started_at": "2026-10-21T06:11:34.148235+00:00", "success": false, "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 51087, "cache_read": 1099000, "completion": 26248, "prompt": 1154995, "reasoning": 0}, "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.64450475, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "finished_at": "2026-10-21T06:18:34.504243+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2309:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00638435900328, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 29, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-2309__opus-5__a1__20261021T061444322645Z", "solve_latency_s": 230.00638435900328, "started_at": "2026-10-21T06:14:44.322630+00:00", "success": false, "task_id": "pylint-dev__astroid-2309", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22259, "cache_read": 413692, "completion": 11930, "prompt": 436009, "reasoning": 7127}, "upstream_digest": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77", "upstream_image_id": "sha256:424e040244842e8181d064a0e84bc67f60c86fceed052e2e3edf84bbd9940c77"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1666", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5550333, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.54369581699939, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "finished_at": "2026-10-21T06:21:26.049958+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1666:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 290.00907700800235, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:f949a914a72f9c7c923de4e51b48eb45b3976e9ea4a3fa4ede56a861e8fd6bbc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1666__sonnet-5__a1__20261021T061614312013Z", "solve_latency_s": 290.00907700800235, "started_at": "2026-10-21T06:16:14.311997+00:00", "success": false, "task_id": "pvlib__pvlib-python-1666", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 46207, "cache_read": 1151729, "completion": 20897, "prompt": 1198036, "reasoning": 11581}, "upstream_digest": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781", "upstream_image_id": "sha256:9d6bc630e6a35fb86aba06246fc8de95fbf527ab66eed0751718b8a7c6298781"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "latency_s": 42.26520768800037, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "latency_s": 41.60962831600045, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:2d654e4f952f8aab69fcc67a0f0558b100033da20ff81f58430493af216291ba", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1365", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "latency_s": 37.42055031199925, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.12878715, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.432110554000246, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "finished_at": "2026-10-21T06:26:14.194867+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00872880399766, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 45, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1978__haiku-4.5__a1__20261021T062352591716Z", "solve_latency_s": 120.00872880399766, "started_at": "2026-10-21T06:23:52.591693+00:00", "success": false, "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18261, "cache_read": 377229, "completion": 7221, "prompt": 427623, "reasoning": 0}, "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.039051100000000005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.536053373001778, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "finished_at": "2026-10-21T06:27:15.945861+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.002291953001986, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 12, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1978__sonnet-5__a1__20261021T062614202893Z", "solve_latency_s": 40.002291953001986, "started_at": "2026-10-21T06:26:14.202878+00:00", "success": false, "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 5999, "cache_read": 47228, "completion": 1456, "prompt": 53251, "reasoning": 157}, "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.76345675, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "finished_at": "2026-10-21T06:28:32.730831+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.00734789200214, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-933__opus-5__a1__20261021T062352595828Z", "solve_latency_s": 280.00734789200214, "started_at": "2026-10-21T06:23:52.595808+00:00", "success": false, "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17933, "cache_read": 91051, "completion": 24230, "prompt": 109004, "reasoning": 21276}, "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1721697, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "finished_at": "2026-10-21T06:30:42.878170+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00383345900264, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-933__haiku-4.5__a1__20261021T062832739377Z", "solve_latency_s": 130.00383345900264, "started_at": "2026-10-21T06:28:32.739361+00:00", "success": false, "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28608, "cache_read": 706207, "completion": 9942, "prompt": 750894, "reasoning": 0}, "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.68405275, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "finished_at": "2026-10-21T06:31:46.089536+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1978:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 270.0068097680014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 15, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1978__opus-5__a1__20261021T062715954897Z", "solve_latency_s": 270.0068097680014, "started_at": "2026-10-21T06:27:15.954882+00:00", "success": false, "task_id": "pylint-dev__astroid-1978", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 14751, "cache_read": 128468, "completion": 21099, "prompt": 143249, "reasoning": 17506}, "upstream_digest": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685", "upstream_image_id": "sha256:c8c32dfa1bb2ac5bf5b52e2c96fbd18b0a3a8a864b30e757b03f5a79a6e6a685"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-933", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2785342, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.521670827998605, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "finished_at": "2026-10-21T06:33:31.557766+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-933:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00452539499747, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 30, "overlay_image_id": "sha256:7ab87185964e24336be8eafa73b6bbe1a076687275cf5420b381c8320b2c4722", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-933__sonnet-5__a1__20261021T063042889310Z", "solve_latency_s": 150.00452539499747, "started_at": "2026-10-21T06:30:42.889294+00:00", "success": true, "task_id": "pydicom__pydicom-933", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27446, "cache_read": 459296, "completion": 11794, "prompt": 486802, "reasoning": 5417}, "upstream_digest": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346", "upstream_image_id": "sha256:375360b5ff24b95ddf37490fa4370b1ec720d0753136ee539a3c4652320aa346"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "latency_s": 53.46030479900219, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4414", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "latency_s": 39.439905870000075, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f363adc31fcd0fd2a64502cf50bea26237876de8451c0066ee416aced97ab0fc", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1048", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "latency_s": 36.34352293600023, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.633041, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "finished_at": "2026-10-21T06:39:40.710035+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0060526820016, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 22, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1333__opus-5__a1__20261021T063630543785Z", "solve_latency_s": 190.0060526820016, "started_at": "2026-10-21T06:36:30.543760+00:00", "success": false, "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24306, "cache_read": 309167, "completion": 13053, "prompt": 333517, "reasoning": 9773}, "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.18808655, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "finished_at": "2026-10-21T06:42:20.906468+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.004997172, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1333__haiku-4.5__a1__20261021T063940718628Z", "solve_latency_s": 160.004997172, "started_at": "2026-10-21T06:39:40.718612+00:00", "success": false, "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 32485, "cache_read": 732533, "completion": 13084, "prompt": 773825, "reasoning": 0}, "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.07471950000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.579913778001355, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "finished_at": "2026-10-21T06:43:39.651573+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1333:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.00300859399795, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 18, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1333__sonnet-5__a1__20261021T064220926048Z", "solve_latency_s": 60.00300859399795, "started_at": "2026-10-21T06:42:20.926031+00:00", "success": true, "task_id": "pylint-dev__astroid-1333", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10423, "cache_read": 115900, "completion": 2541, "prompt": 126359, "reasoning": 495}, "upstream_digest": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4", "upstream_image_id": "sha256:4014d41119b96aa182d1c2b1299c3ec32a6b36ea9ef03e8e90f053b4a535b2b4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "latency_s": 44.45388161699884, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-3675", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1469", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1469:latest", "latency_s": 54.79591803199946, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1469", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e", "upstream_image_id": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "latency_s": 38.184336573998735, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "sqlfluff__sqlfluff-905", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "upstream_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.59255725, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "finished_at": "2026-10-21T06:56:23.758012+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.00510407599722, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 21, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "sqlfluff__sqlfluff-905__opus-5__a1__20261021T065313604865Z", "solve_latency_s": 190.00510407599722, "started_at": "2026-10-21T06:53:13.604848+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-905", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26169, "cache_read": 301282, "completion": 11126, "prompt": 327493, "reasoning": 6556}, "upstream_digest": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "upstream_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1469", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.567756, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "finished_at": "2026-10-21T06:57:13.797849+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1469:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.01594634999856, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1469__sonnet-5__a1__20261021T065313600905Z", "solve_latency_s": 240.01594634999856, "started_at": "2026-10-21T06:53:13.600884+00:00", "success": false, "task_id": "pvlib__pvlib-python-1469", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 44296, "cache_read": 1284680, "completion": 19988, "prompt": 1329076, "reasoning": 4162}, "upstream_digest": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e", "upstream_image_id": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.17876025, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "finished_at": "2026-10-21T06:58:53.965868+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00807084600092, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "sqlfluff__sqlfluff-905__haiku-4.5__a1__20261021T065623768360Z", "solve_latency_s": 150.00807084600092, "started_at": "2026-10-21T06:56:23.768341+00:00", "success": false, "task_id": "sqlfluff__sqlfluff-905", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31279, "cache_read": 774935, "completion": 11115, "prompt": 812807, "reasoning": 0}, "upstream_digest": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "upstream_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1469", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.550582, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "finished_at": "2026-10-21T06:59:33.980090+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1469:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.01173601799746, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 15, "overlay_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1469__opus-5__a1__20261021T065713813256Z", "solve_latency_s": 140.01173601799746, "started_at": "2026-10-21T06:57:13.813239+00:00", "success": false, "task_id": "pvlib__pvlib-python-1469", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33250, "cache_read": 218489, "completion": 9335, "prompt": 251769, "reasoning": 6505}, "upstream_digest": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e", "upstream_image_id": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1469", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.1805501, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "finished_at": "2026-10-21T07:01:54.132385+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1469:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.0037113469989, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:654098681bceddc57ea4e52b3eec5a5a0f91fac1cc86d99fdabe58f69d304d07", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1469__haiku-4.5__a1__20261021T065933989226Z", "solve_latency_s": 140.0037113469989, "started_at": "2026-10-21T06:59:33.989210+00:00", "success": false, "task_id": "pvlib__pvlib-python-1469", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31836, "cache_read": 723171, "completion": 10743, "prompt": 769730, "reasoning": 0}, "upstream_digest": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e", "upstream_image_id": "sha256:f7546e4ee6c4da69d216ae7b3696fffa219826722aef4608d136715120f1359e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2692213, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.044535684999573, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "finished_at": "2026-10-21T07:01:55.165732+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.sqlfluff_1776_sqlfluff-905:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00485384499916, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 35, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "sqlfluff__sqlfluff-905__sonnet-5__a1__20261021T065853975268Z", "solve_latency_s": 160.00485384499916, "started_at": "2026-10-21T06:58:53.975250+00:00", "success": true, "task_id": "sqlfluff__sqlfluff-905", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28699, "cache_read": 556769, "completion": 8598, "prompt": 585538, "reasoning": 3151}, "upstream_digest": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133", "upstream_image_id": "sha256:c97546dcb931d0d34dec341469221f13206db0dcce47afef9898bcd299f26133"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1186", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1186:latest", "latency_s": 43.30945698800133, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1186", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947", "upstream_image_id": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1764", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1764:latest", "latency_s": 47.19117799599917, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1764", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b", "upstream_image_id": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1073", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1073:latest", "latency_s": 43.99605777300167, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1073", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be", "upstream_image_id": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1073", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.44036600000000004, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 24.423070098000608, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "finished_at": "2026-10-21T07:07:52.176664+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1073:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.0086774320007, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 15, "overlay_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1073__opus-5__a1__20261021T070517575210Z", "solve_latency_s": 130.0086774320007, "started_at": "2026-10-21T07:05:17.575190+00:00", "success": true, "task_id": "pvlib__pvlib-python-1073", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22404, "cache_read": 188482, "completion": 8238, "prompt": 210916, "reasoning": 3592}, "upstream_digest": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be", "upstream_image_id": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1073", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.1474046, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "finished_at": "2026-10-21T07:09:32.333199+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1073:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.0032186899989, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1073__haiku-4.5__a1__20261021T070752186717Z", "solve_latency_s": 100.0032186899989, "started_at": "2026-10-21T07:07:52.186702+00:00", "success": false, "task_id": "pvlib__pvlib-python-1073", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27244, "cache_read": 635426, "completion": 6500, "prompt": 679977, "reasoning": 0}, "upstream_digest": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be", "upstream_image_id": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1764", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6227719, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "finished_at": "2026-10-21T07:09:37.781514+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1764:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 260.0089212240018, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1764__sonnet-5__a1__20261021T070517584904Z", "solve_latency_s": 260.0089212240018, "started_at": "2026-10-21T07:05:17.584890+00:00", "success": false, "task_id": "pvlib__pvlib-python-1764", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 65503, "cache_read": 1833322, "completion": 9215, "prompt": 1898925, "reasoning": 2174}, "upstream_digest": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b", "upstream_image_id": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1186", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2692488, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "finished_at": "2026-10-21T07:09:37.797284+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1186:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 260.01059098300175, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1186__haiku-4.5__a1__20261021T070517580705Z", "solve_latency_s": 260.01059098300175, "started_at": "2026-10-21T07:05:17.580685+00:00", "success": false, "task_id": "pvlib__pvlib-python-1186", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 49180, "cache_read": 1110828, "completion": 18446, "prompt": 1164469, "reasoning": 0}, "upstream_digest": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947", "upstream_image_id": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1073", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2132612, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 22.86541775599835, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "finished_at": "2026-10-21T07:11:35.374628+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1073:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.0061386410016, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 33, "overlay_image_id": "sha256:d25858fb769c2083ac2e9babc601811b2de28f0b5fc52870ce7952c3b50705b3", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1073__sonnet-5__a1__20261021T070932343593Z", "solve_latency_s": 100.0061386410016, "started_at": "2026-10-21T07:09:32.343577+00:00", "success": false, "task_id": "pvlib__pvlib-python-1073", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24324, "cache_read": 434446, "completion": 6543, "prompt": 458836, "reasoning": 347}, "upstream_digest": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be", "upstream_image_id": "sha256:ec453a3e6c2e920955ef8530d7f0ad6e07fa32729c24ba045b8d6058a8c7f6be"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1764", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6090315000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "finished_at": "2026-10-21T07:12:27.976663+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1764:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.01246403199912, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 17, "overlay_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1764__opus-5__a1__20261021T070937796545Z", "solve_latency_s": 170.01246403199912, "started_at": "2026-10-21T07:09:37.796526+00:00", "success": false, "task_id": "pvlib__pvlib-python-1764", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28078, "cache_read": 287298, "completion": 11589, "prompt": 315410, "reasoning": 9131}, "upstream_digest": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b", "upstream_image_id": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1186", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5616252, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "finished_at": "2026-10-21T07:12:47.980692+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1186:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.00435195500177, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1186__sonnet-5__a1__20261021T070937808014Z", "solve_latency_s": 190.00435195500177, "started_at": "2026-10-21T07:09:37.807996+00:00", "success": false, "task_id": "pvlib__pvlib-python-1186", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 54518, "cache_read": 1557201, "completion": 11369, "prompt": 1611819, "reasoning": 4361}, "upstream_digest": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947", "upstream_image_id": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1186", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.58495025, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "finished_at": "2026-10-21T07:15:38.163740+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1186:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.00410956500127, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 17, "overlay_image_id": "sha256:a8fffafbc2f7fb2ecc8ea9f67fb61a2e392dc2e5413cfde65c11fd5a340a7c6d", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1186__opus-5__a1__20261021T071247991297Z", "solve_latency_s": 170.00410956500127, "started_at": "2026-10-21T07:12:47.991279+00:00", "success": false, "task_id": "pvlib__pvlib-python-1186", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28167, "cache_read": 255073, "completion": 11248, "prompt": 283274, "reasoning": 8113}, "upstream_digest": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947", "upstream_image_id": "sha256:95d5a4462be39cfa0d317135f0ab8c485072ba3746d94588f277e43e2c050947"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1764", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2645329, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "finished_at": "2026-10-21T07:15:48.141463+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1764:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00787617999958, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:552a1fa28e2616f355e2152c80d3d2c4d705ca62f37bb32839ccc49c54e552ba", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1764__haiku-4.5__a1__20261021T071227987605Z", "solve_latency_s": 200.00787617999958, "started_at": "2026-10-21T07:12:27.987589+00:00", "success": false, "task_id": "pvlib__pvlib-python-1764", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 39276, "cache_read": 1010279, "completion": 19125, "prompt": 1068340, "reasoning": 0}, "upstream_digest": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b", "upstream_image_id": "sha256:ebbff79d8f3fb08a515f8b7a76b033aebaef4a3e0393f02fc8640aaa179d354b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1852", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1852:latest", "latency_s": 46.35951804200158, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1852", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997", "upstream_image_id": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "latency_s": 42.63889835000009, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1702", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "upstream_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-907", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-907:latest", "latency_s": 46.23383768500207, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-907", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f", "upstream_image_id": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.073674, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.238022716002888, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "finished_at": "2026-10-21T07:20:03.212578+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.003130209999654, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 14, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1702__sonnet-5__a1__20261021T071903840115Z", "solve_latency_s": 40.003130209999654, "started_at": "2026-10-21T07:19:03.840101+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1702", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 12526, "cache_read": 99365, "completion": 2243, "prompt": 111919, "reasoning": 202}, "upstream_digest": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "upstream_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-907", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5930905, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "finished_at": "2026-10-21T07:22:23.863749+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-907:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.0119838419996, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 13, "overlay_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-907__opus-5__a1__20261021T071903835205Z", "solve_latency_s": 200.0119838419996, "started_at": "2026-10-21T07:19:03.835189+00:00", "success": false, "task_id": "pvlib__pvlib-python-907", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26542, "cache_read": 197096, "completion": 13141, "prompt": 223664, "reasoning": 10162}, "upstream_digest": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f", "upstream_image_id": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1852", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.23729935, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 22.70222214900059, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "finished_at": "2026-10-21T07:22:26.716133+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1852:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.0175402829991, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1852__haiku-4.5__a1__20261021T071903828893Z", "solve_latency_s": 180.0175402829991, "started_at": "2026-10-21T07:19:03.828870+00:00", "success": false, "task_id": "pvlib__pvlib-python-1852", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41255, "cache_read": 975616, "completion": 14607, "prompt": 1032005, "reasoning": 0}, "upstream_digest": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997", "upstream_image_id": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.66759075, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "finished_at": "2026-10-21T07:23:53.401215+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00675784399937, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1702__opus-5__a1__20261021T072003224252Z", "solve_latency_s": 230.00675784399937, "started_at": "2026-10-21T07:20:03.224237+00:00", "success": false, "task_id": "marshmallow-code__marshmallow-1702", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 21033, "cache_read": 198149, "completion": 17476, "prompt": 219214, "reasoning": 13705}, "upstream_digest": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "upstream_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1852", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2139996, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 23.707458058997872, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "finished_at": "2026-10-21T07:24:30.591962+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1852:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00324067700058, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 23, "overlay_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1852__sonnet-5__a1__20261021T072226728112Z", "solve_latency_s": 100.00324067700058, "started_at": "2026-10-21T07:22:26.728093+00:00", "success": false, "task_id": "pvlib__pvlib-python-1852", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24622, "cache_read": 346613, "completion": 8303, "prompt": 371281, "reasoning": 526}, "upstream_digest": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997", "upstream_image_id": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-907", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.21830615, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.203247146997455, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "finished_at": "2026-10-21T07:25:45.308442+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-907:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00790279400098, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 45, "overlay_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-907__haiku-4.5__a1__20261021T072223874566Z", "solve_latency_s": 180.00790279400098, "started_at": "2026-10-21T07:22:23.874550+00:00", "success": false, "task_id": "pvlib__pvlib-python-907", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 40969, "cache_read": 891799, "completion": 14630, "prompt": 937533, "reasoning": 0}, "upstream_digest": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f", "upstream_image_id": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.20312645000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.9953326979994, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "finished_at": "2026-10-21T07:26:13.556316+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1702:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00615079300042, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 46, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1702__haiku-4.5__a1__20261021T072353412468Z", "solve_latency_s": 120.00615079300042, "started_at": "2026-10-21T07:23:53.412450+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1702", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 42073, "cache_read": 953572, "completion": 8109, "prompt": 1010278, "reasoning": 0}, "upstream_digest": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371", "upstream_image_id": "sha256:fa58cabe15f5dfa86ddfb66c3834c63ea73f0d507cb44cc59a42902ceaddb371"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1852", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.46864500000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 24.29761684099867, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "finished_at": "2026-10-21T07:26:45.183802+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1852:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.00385048799944, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 17, "overlay_image_id": "sha256:89d911c716cc2e3027946f83d3de048bb605ac4d73db1a7d3ae44df7fa783817", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1852__opus-5__a1__20261021T072430604878Z", "solve_latency_s": 110.00385048799944, "started_at": "2026-10-21T07:24:30.604858+00:00", "success": true, "task_id": "pvlib__pvlib-python-1852", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25340, "cache_read": 251050, "completion": 7383, "prompt": 276424, "reasoning": 2607}, "upstream_digest": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997", "upstream_image_id": "sha256:a3529b0e0dedbb173e0dc728e906c2ca6d5c3a6bde5a38b21a7d823f30ed7997"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-907", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.4458437, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "finished_at": "2026-10-21T07:28:45.524135+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-907:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00569227400047, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:c3648c4c65131d7032a48849ba66eb8d18cfb8464b16b4ddb68d1d468b1ecb71", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-907__sonnet-5__a1__20261021T072545320728Z", "solve_latency_s": 180.00569227400047, "started_at": "2026-10-21T07:25:45.320712+00:00", "success": false, "task_id": "pvlib__pvlib-python-907", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 44199, "cache_read": 1064881, "completion": 12217, "prompt": 1109180, "reasoning": 3527}, "upstream_digest": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f", "upstream_image_id": "sha256:40cac01d684f86ea0afe064f8900826619df27988f281e3461e2f127dfcdc45f"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest", "latency_s": 38.152053487003286, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-432", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21", "upstream_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1017", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1017:latest", "latency_s": 40.659150239000155, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1017", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e", "upstream_image_id": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1241", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1241:latest", "latency_s": 37.760334227001294, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1241", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb", "upstream_image_id": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1017", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.4858933, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "finished_at": "2026-10-21T07:34:53.904737+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1017:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0119462980001, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1017__sonnet-5__a1__20261021T073143706161Z", "solve_latency_s": 190.0119462980001, "started_at": "2026-10-21T07:31:43.706138+00:00", "success": false, "task_id": "pydicom__pydicom-1017", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 44371, "cache_read": 1183579, "completion": 13805, "prompt": 1228050, "reasoning": 4318}, "upstream_digest": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e", "upstream_image_id": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1241", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6233847499999999, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "finished_at": "2026-10-21T07:35:03.736057+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1241:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00757747799798, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 19, "overlay_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1241__opus-5__a1__20261021T073143711303Z", "solve_latency_s": 200.00757747799798, "started_at": "2026-10-21T07:31:43.711280+00:00", "success": false, "task_id": "pydicom__pydicom-1241", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23325, "cache_read": 270777, "completion": 13681, "prompt": 294140, "reasoning": 8769}, "upstream_digest": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb", "upstream_image_id": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1017", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5885055, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "finished_at": "2026-10-21T07:37:34.118427+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1017:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00947596100013, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 13, "overlay_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1017__opus-5__a1__20261021T073453918326Z", "solve_latency_s": 160.00947596100013, "started_at": "2026-10-21T07:34:53.918308+00:00", "success": false, "task_id": "pydicom__pydicom-1017", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 29656, "cache_read": 201051, "completion": 12100, "prompt": 230733, "reasoning": 9271}, "upstream_digest": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e", "upstream_image_id": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1241", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2829628, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "finished_at": "2026-10-21T07:38:23.900455+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1241:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00729631500144, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1241__haiku-4.5__a1__20261021T073503747908Z", "solve_latency_s": 200.00729631500144, "started_at": "2026-10-21T07:35:03.747891+00:00", "success": false, "task_id": "pydicom__pydicom-1241", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 47232, "cache_read": 1229248, "completion": 18963, "prompt": 1282663, "reasoning": 0}, "upstream_digest": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb", "upstream_image_id": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1017", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.1607172, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "finished_at": "2026-10-21T07:39:04.314772+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1017:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00373087299886, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:0366d77de31028070b7fd292cf9335b75b0988fb0d4a127716d832d42a086e21", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1017__haiku-4.5__a1__20261021T073734130706Z", "solve_latency_s": 90.00373087299886, "started_at": "2026-10-21T07:37:34.130688+00:00", "success": false, "task_id": "pydicom__pydicom-1017", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31688, "cache_read": 792502, "completion": 5836, "prompt": 836867, "reasoning": 0}, "upstream_digest": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e", "upstream_image_id": "sha256:625dd85c76b9f1d149b0f3b7272b3fdafabfe0940ad48116893d2c29633ac31e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1241", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.3019577, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.6398263580013, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "finished_at": "2026-10-21T07:40:42.684659+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1241:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00659115300004, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 34, "overlay_image_id": "sha256:9b1e6d05e4d5588946d33924ae83cf5ba3dc6b0f675be5da7bdb6183ca878959", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1241__sonnet-5__a1__20261021T073823913897Z", "solve_latency_s": 120.00659115300004, "started_at": "2026-10-21T07:38:23.913882+00:00", "success": true, "task_id": "pydicom__pydicom-1241", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 34395, "cache_read": 636871, "completion": 8846, "prompt": 671334, "reasoning": 1862}, "upstream_digest": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb", "upstream_image_id": "sha256:bfe43b77dc3b46d085a26de795d5e759b88e5cc9fcb09fbd87ebc3152c8c4cfb"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428", "category": "feature_implementation", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:68636bf54706b0f82a31980d5cb719e6056f8129a4d79f3d3a47f67f478f2e49", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "latency_s": 49.36457281099865, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:68636bf54706b0f82a31980d5cb719e6056f8129a4d79f3d3a47f67f478f2e49", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1428", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1228", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1228:latest", "latency_s": 37.49365611699977, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1228", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01", "upstream_image_id": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-718", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-718:latest", "latency_s": 46.919107115998486, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-718", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef", "upstream_image_id": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1228", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2716825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.538643478997983, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "finished_at": "2026-10-21T08:08:22.545292+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1228:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00913361200219, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 35, "overlay_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1228__sonnet-5__a1__20261021T080603864016Z", "solve_latency_s": 120.00913361200219, "started_at": "2026-10-21T08:06:03.863993+00:00", "success": true, "task_id": "pydicom__pydicom-1228", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30423, "cache_read": 555675, "completion": 8435, "prompt": 586168, "reasoning": 1399}, "upstream_digest": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01", "upstream_image_id": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-718", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.57999475, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "finished_at": "2026-10-21T08:08:34.064669+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-718:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00590279600146, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 19, "overlay_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-718__opus-5__a1__20261021T080603869357Z", "solve_latency_s": 150.00590279600146, "started_at": "2026-10-21T08:06:03.869340+00:00", "success": false, "task_id": "pvlib__pvlib-python-718", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30517, "cache_read": 321797, "completion": 9127, "prompt": 352352, "reasoning": 5230}, "upstream_digest": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef", "upstream_image_id": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1228", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6069405, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "finished_at": "2026-10-21T08:11:12.721822+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1228:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.0085470000049, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 21, "overlay_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1228__opus-5__a1__20261021T080822558764Z", "solve_latency_s": 170.0085470000049, "started_at": "2026-10-21T08:08:22.558749+00:00", "success": false, "task_id": "pydicom__pydicom-1228", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26488, "cache_read": 333661, "completion": 10974, "prompt": 360191, "reasoning": 4760}, "upstream_digest": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01", "upstream_image_id": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-718", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.4497295, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "finished_at": "2026-10-21T08:14:54.263850+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-718:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 380.0103277410017, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-718__haiku-4.5__a1__20261021T080834080762Z", "solve_latency_s": 380.0103277410017, "started_at": "2026-10-21T08:08:34.080742+00:00", "success": false, "task_id": "pvlib__pvlib-python-718", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 71560, "cache_read": 1605535, "completion": 37797, "prompt": 1687836, "reasoning": 0}, "upstream_digest": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef", "upstream_image_id": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1228", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2876862, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "finished_at": "2026-10-21T08:15:02.955222+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1228:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00664918999973, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:57d2a4b0e830734ccb21ee9a7c4316be678337381c1192c26d4a96e452af4e0a", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1228__haiku-4.5__a1__20261021T081112735083Z", "solve_latency_s": 230.00664918999973, "started_at": "2026-10-21T08:11:12.735066+00:00", "success": false, "task_id": "pydicom__pydicom-1228", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 45498, "cache_read": 1174727, "completion": 19766, "prompt": 1234736, "reasoning": 0}, "upstream_digest": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01", "upstream_image_id": "sha256:7eaae4035199b99f58cc02fbb4ef459e505aee432fc2531c3a876134572d5b01"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-718", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5498171000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "finished_at": "2026-10-21T08:18:24.506430+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-718:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.0076161190009, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:00656eccc882ccff73b2da099b7db38005a91d926c9adbb2613b81560a85e586", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-718__sonnet-5__a1__20261021T081454277761Z", "solve_latency_s": 210.0076161190009, "started_at": "2026-10-21T08:14:54.277742+00:00", "success": false, "task_id": "pvlib__pvlib-python-718", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 50949, "cache_read": 1477023, "completion": 12684, "prompt": 1528072, "reasoning": 6143}, "upstream_digest": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef", "upstream_image_id": "sha256:67ef83e61c37304a0608b4c4963296c2442ef4b1ce0723183eb329f21619bcef"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1176", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1176:latest", "latency_s": 45.82073983800365, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1176", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac", "upstream_image_id": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1181", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1181:latest", "latency_s": 46.233432954999444, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1181", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9", "upstream_image_id": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "latency_s": 37.918114620995766, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1616", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "upstream_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.26296425, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.187045872000454, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "finished_at": "2026-10-21T08:23:59.484911+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00416999200388, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 14, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1616__opus-5__a1__20261021T082140135380Z", "solve_latency_s": 120.00416999200388, "started_at": "2026-10-21T08:21:40.135365+00:00", "success": true, "task_id": "pylint-dev__astroid-1616", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10929, "cache_read": 86236, "completion": 6056, "prompt": 97193, "reasoning": 3123}, "upstream_digest": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "upstream_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1181", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5308269, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 23.84915690500202, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "finished_at": "2026-10-21T08:25:04.130435+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1181:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00894689399865, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 46, "overlay_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1181__sonnet-5__a1__20261021T082140129550Z", "solve_latency_s": 180.00894689399865, "started_at": "2026-10-21T08:21:40.129532+00:00", "success": false, "task_id": "pvlib__pvlib-python-1181", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 54797, "cache_read": 1234852, "completion": 14668, "prompt": 1289741, "reasoning": 1523}, "upstream_digest": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9", "upstream_image_id": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1176", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2802225, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "finished_at": "2026-10-21T08:25:10.163441+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1176:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.0144302810004, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1176__haiku-4.5__a1__20261021T082140123909Z", "solve_latency_s": 210.0144302810004, "started_at": "2026-10-21T08:21:40.123884+00:00", "success": false, "task_id": "pvlib__pvlib-python-1176", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 47634, "cache_read": 1236190, "completion": 17311, "prompt": 1294330, "reasoning": 0}, "upstream_digest": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac", "upstream_image_id": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2220352, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "finished_at": "2026-10-21T08:26:39.727780+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00602080299723, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1616__haiku-4.5__a1__20261021T082359503512Z", "solve_latency_s": 160.00602080299723, "started_at": "2026-10-21T08:23:59.503493+00:00", "success": false, "task_id": "pylint-dev__astroid-1616", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37360, "cache_read": 981712, "completion": 12412, "prompt": 1034176, "reasoning": 0}, "upstream_digest": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "upstream_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1181", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.56541625, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "finished_at": "2026-10-21T08:27:14.326468+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1181:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00423074699938, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 15, "overlay_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1181__opus-5__a1__20261021T082504146707Z", "solve_latency_s": 130.00423074699938, "started_at": "2026-10-21T08:25:04.146688+00:00", "success": false, "task_id": "pvlib__pvlib-python-1181", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31945, "cache_read": 266070, "completion": 9303, "prompt": 298045, "reasoning": 4428}, "upstream_digest": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9", "upstream_image_id": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1176", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6384284, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "finished_at": "2026-10-21T08:28:10.393304+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1176:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.0093161640034, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1176__sonnet-5__a1__20261021T082510177792Z", "solve_latency_s": 180.0093161640034, "started_at": "2026-10-21T08:25:10.177774+00:00", "success": false, "task_id": "pvlib__pvlib-python-1176", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 63332, "cache_read": 1825992, "completion": 11470, "prompt": 1889424, "reasoning": 5292}, "upstream_digest": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac", "upstream_image_id": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.080498, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.47246191299928, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "finished_at": "2026-10-21T08:28:28.379020+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1616:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00671298499947, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 18, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1616__sonnet-5__a1__20261021T082639752994Z", "solve_latency_s": 90.00671298499947, "started_at": "2026-10-21T08:26:39.752972+00:00", "success": true, "task_id": "pylint-dev__astroid-1616", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10788, "cache_read": 107030, "completion": 3205, "prompt": 117854, "reasoning": 477}, "upstream_digest": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e", "upstream_image_id": "sha256:4685f3bf018042458da5aa31cbb06cca716f526a4ff2c23e095630328402b90e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1181", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.16953959999999998, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "finished_at": "2026-10-21T08:29:04.504467+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1181:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.00331660699885, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:f4957482cd91bd29d645000cadd7fc847b892a432efa24ca1534f98801afb691", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1181__haiku-4.5__a1__20261021T082714340466Z", "solve_latency_s": 110.00331660699885, "started_at": "2026-10-21T08:27:14.340446+00:00", "success": false, "task_id": "pvlib__pvlib-python-1181", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26230, "cache_read": 818851, "completion": 8674, "prompt": 856578, "reasoning": 0}, "upstream_digest": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9", "upstream_image_id": "sha256:58f5f77abfa935947de2dde70e380f5c158b042d845f73a420564153e2fd03c9"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1176", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.57946375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "finished_at": "2026-10-21T08:30:30.563814+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1176:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.00373233899882, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 14, "overlay_image_id": "sha256:1664f2bfcc38a233c34bdd63782403bf299a7ae13c55b7e56c341f79467109af", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1176__opus-5__a1__20261021T082810407848Z", "solve_latency_s": 140.00373233899882, "started_at": "2026-10-21T08:28:10.407832+00:00", "success": false, "task_id": "pvlib__pvlib-python-1176", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 34797, "cache_read": 237135, "completion": 9731, "prompt": 271960, "reasoning": 7464}, "upstream_digest": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac", "upstream_image_id": "sha256:244fb7a9343866ca8a251c50bb617028f28fb5ffc9ffbb57de7284ed774b25ac"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-823", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-823:latest", "latency_s": 42.620264132005104, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-823", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b", "upstream_image_id": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1478", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1478:latest", "latency_s": 57.20094211900141, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1478", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190", "upstream_image_id": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "latency_s": 36.72900991899951, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1417", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "upstream_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.619361, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "finished_at": "2026-10-21T08:36:51.039758+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0047424709992, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 12, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1417__opus-5__a1__20261021T083340899433Z", "solve_latency_s": 190.0047424709992, "started_at": "2026-10-21T08:33:40.899419+00:00", "success": false, "task_id": "pylint-dev__astroid-1417", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26356, "cache_read": 156982, "completion": 15041, "prompt": 183362, "reasoning": 12434}, "upstream_digest": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "upstream_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1478", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5461759, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "finished_at": "2026-10-21T08:37:41.039127+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1478:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.01104093199683, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1478__sonnet-5__a1__20261021T083340894250Z", "solve_latency_s": 240.01104093199683, "started_at": "2026-10-21T08:33:40.894230+00:00", "success": false, "task_id": "pvlib__pvlib-python-1478", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 39139, "cache_read": 1080992, "completion": 23193, "prompt": 1120231, "reasoning": 11679}, "upstream_digest": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190", "upstream_image_id": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.19515495, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "finished_at": "2026-10-21T08:39:21.184967+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00514435600053, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1417__haiku-4.5__a1__20261021T083651056862Z", "solve_latency_s": 150.00514435600053, "started_at": "2026-10-21T08:36:51.056843+00:00", "success": false, "task_id": "pylint-dev__astroid-1417", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 38947, "cache_read": 904732, "completion": 10330, "prompt": 948027, "reasoning": 0}, "upstream_digest": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "upstream_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1478", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6376850000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "finished_at": "2026-10-21T08:40:31.246921+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1478:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.00419574099942, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1478__opus-5__a1__20261021T083741053527Z", "solve_latency_s": 170.00419574099942, "started_at": "2026-10-21T08:37:41.053510+00:00", "success": false, "task_id": "pvlib__pvlib-python-1478", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 34354, "cache_read": 249025, "completion": 11932, "prompt": 283411, "reasoning": 5505}, "upstream_digest": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190", "upstream_image_id": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-823", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.48201065, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "finished_at": "2026-10-21T08:40:41.038588+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-823:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 420.01377802599745, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-823__haiku-4.5__a1__20261021T083340888304Z", "solve_latency_s": 420.01377802599745, "started_at": "2026-10-21T08:33:40.888283+00:00", "success": false, "task_id": "pvlib__pvlib-python-823", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 72753, "cache_read": 1809354, "completion": 40451, "prompt": 1889986, "reasoning": 0}, "upstream_digest": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b", "upstream_image_id": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.4203287, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "finished_at": "2026-10-21T08:41:51.378614+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1417:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.0062031540001, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1417__sonnet-5__a1__20261021T083921202366Z", "solve_latency_s": 150.0062031540001, "started_at": "2026-10-21T08:39:21.202345+00:00", "success": false, "task_id": "pylint-dev__astroid-1417", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37013, "cache_read": 983181, "completion": 13096, "prompt": 1020294, "reasoning": 2765}, "upstream_digest": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d", "upstream_image_id": "sha256:4bebee732bc6ab0ab8ae6a09ad0cf80d224c153c54bdc29c38a00afab3a1ea8d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1478", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.17128184999999999, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "finished_at": "2026-10-21T08:42:21.414624+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1478:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.0059342589957, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:6431207594d0d9ec61858c522c64e546a068396766a9e34a38e67fcad6afea63", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1478__haiku-4.5__a1__20261021T084031264087Z", "solve_latency_s": 110.0059342589957, "started_at": "2026-10-21T08:40:31.264067+00:00", "success": false, "task_id": "pvlib__pvlib-python-1478", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31787, "cache_read": 853871, "completion": 7894, "prompt": 892349, "reasoning": 0}, "upstream_digest": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190", "upstream_image_id": "sha256:a10ea1edb6d166be1e47a756d5c3e70071672609ed4e94ce448d806b7270c190"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-823", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6774091, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "finished_at": "2026-10-21T08:44:31.262208+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-823:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00624105399766, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-823__sonnet-5__a1__20261021T084041053838Z", "solve_latency_s": 230.00624105399766, "started_at": "2026-10-21T08:40:41.053819+00:00", "success": false, "task_id": "pvlib__pvlib-python-823", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 67995, "cache_read": 1843808, "completion": 13846, "prompt": 1911903, "reasoning": 7597}, "upstream_digest": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b", "upstream_image_id": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-823", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6217145, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "finished_at": "2026-10-21T08:48:01.460216+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-823:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.00741883299634, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 11, "overlay_image_id": "sha256:6cbd5c0b736f5a169763a8d9922f81a0fa60ee0512f709a819fd264841988713", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-823__opus-5__a1__20261021T084431279301Z", "solve_latency_s": 210.00741883299634, "started_at": "2026-10-21T08:44:31.279281+00:00", "success": false, "task_id": "pvlib__pvlib-python-823", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22672, "cache_read": 137259, "completion": 16451, "prompt": 159953, "reasoning": 14617}, "upstream_digest": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b", "upstream_image_id": "sha256:6d83008aabb35cc974aba8acf532243c9cd7cf28157b63005e2d22f96d718a0b"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1682", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1682:latest", "latency_s": 43.40051622399187, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1682", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df", "upstream_image_id": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "latency_s": 67.93332233199908, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4648", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "latency_s": 44.1322110139954, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4311", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1682", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2680082, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "finished_at": "2026-10-21T09:11:43.456688+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1682:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.00784773399937, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1682__haiku-4.5__a1__20261021T090833316091Z", "solve_latency_s": 190.00784773399937, "started_at": "2026-10-21T09:08:33.316067+00:00", "success": false, "task_id": "pvlib__pvlib-python-1682", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 51696, "cache_read": 1117602, "completion": 17536, "prompt": 1173246, "reasoning": 0}, "upstream_digest": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df", "upstream_image_id": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1682", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.7096772, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "TimeExceeded", "final_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "finished_at": "2026-10-21T09:21:33.677529+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1682:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 590.0152349119962, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 37, "overlay_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1682__sonnet-5__a1__20261021T091143472744Z", "solve_latency_s": 590.0152349119962, "started_at": "2026-10-21T09:11:43.472727+00:00", "success": false, "task_id": "pvlib__pvlib-python-1682", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 40074, "cache_read": 844821, "completion": 44038, "prompt": 884969, "reasoning": 33265}, "upstream_digest": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df", "upstream_image_id": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1682", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.57882825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "finished_at": "2026-10-21T09:24:53.884442+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1682:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00584205499763, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 13, "overlay_image_id": "sha256:99d3154618b92c708a6657d6a8ebb3ad2d93814b369869e8c6a62f2e15e02d53", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1682__opus-5__a1__20261021T092133693319Z", "solve_latency_s": 200.00584205499763, "started_at": "2026-10-21T09:21:33.693300+00:00", "success": false, "task_id": "pvlib__pvlib-python-1682", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30479, "cache_read": 180459, "completion": 11919, "prompt": 210964, "reasoning": 8284}, "upstream_digest": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df", "upstream_image_id": "sha256:51210cb96ad70a016dccdd76057a86bc7f62a91746ddfa9de6256794c296b3df"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1426", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1426:latest", "latency_s": 39.99264537199633, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1426", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893", "upstream_image_id": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "latency_s": 38.82054907600104, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-2123", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "upstream_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1458", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1458:latest", "latency_s": 39.70100921500125, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1458", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5", "upstream_image_id": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1458", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.26861050000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.882132279999496, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "finished_at": "2026-10-21T09:29:06.772069+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1458:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.00217850400077, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 15, "overlay_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1458__opus-5__a1__20261021T092746740648Z", "solve_latency_s": 60.00217850400077, "started_at": "2026-10-21T09:27:46.740632+00:00", "success": true, "task_id": "pvlib__pvlib-python-1458", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17402, "cache_read": 143546, "completion": 3517, "prompt": 160978, "reasoning": 1099}, "upstream_digest": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5", "upstream_image_id": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1426", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.1640905, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "finished_at": "2026-10-21T09:29:36.891892+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1426:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.0157040080012, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1426__haiku-4.5__a1__20261021T092746719644Z", "solve_latency_s": 110.0157040080012, "started_at": "2026-10-21T09:27:46.719622+00:00", "success": false, "task_id": "pvlib__pvlib-python-1426", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 32194, "cache_read": 741540, "completion": 8218, "prompt": 782338, "reasoning": 0}, "upstream_digest": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893", "upstream_image_id": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2579592, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.07155434499873, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "finished_at": "2026-10-21T09:30:25.965224+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.00937037500262, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 34, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-2123__sonnet-5__a1__20261021T092746734011Z", "solve_latency_s": 140.00937037500262, "started_at": "2026-10-21T09:27:46.733993+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-2123", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26616, "cache_read": 443866, "completion": 10251, "prompt": 470550, "reasoning": 4106}, "upstream_digest": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "upstream_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1426", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.1588962, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.725819598999806, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "finished_at": "2026-10-21T09:30:56.783088+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1426:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.002241103000415, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 22, "overlay_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1426__sonnet-5__a1__20261021T092936908344Z", "solve_latency_s": 60.002241103000415, "started_at": "2026-10-21T09:29:36.908325+00:00", "success": true, "task_id": "pvlib__pvlib-python-1426", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 26726, "cache_read": 296716, "completion": 3265, "prompt": 323486, "reasoning": 675}, "upstream_digest": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893", "upstream_image_id": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1458", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.14684745, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.716543094997178, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "finished_at": "2026-10-21T09:31:06.653698+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1458:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00288976500451, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 36, "overlay_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1458__haiku-4.5__a1__20261021T092906793399Z", "solve_latency_s": 100.00288976500451, "started_at": "2026-10-21T09:29:06.793383+00:00", "success": true, "task_id": "pvlib__pvlib-python-1458", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 32133, "cache_read": 620822, "completion": 8032, "prompt": 657394, "reasoning": 0}, "upstream_digest": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5", "upstream_image_id": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1458", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.024967800000000002, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.818735610999283, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "finished_at": "2026-10-21T09:31:46.610306+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1458:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 20.00162174199795, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 6, "overlay_image_id": "sha256:ad7e3a313809ec430031d1545acd0fcf0b49141d7e9fd4412e9e7e50b3f280f5", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1458__sonnet-5__a1__20261021T093106670514Z", "solve_latency_s": 20.00162174199795, "started_at": "2026-10-21T09:31:06.670497+00:00", "success": true, "task_id": "pvlib__pvlib-python-1458", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 6130, "cache_read": 16694, "completion": 628, "prompt": 22836, "reasoning": 10}, "upstream_digest": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5", "upstream_image_id": "sha256:e265b7df93ec18c58a24e0a8f249a58910bdaaf6a4ed7e8db1734481166277b5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.383684, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.27930404599465, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "finished_at": "2026-10-21T09:32:45.472222+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00699871999677, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 12, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-2123__opus-5__a1__20261021T093026002514Z", "solve_latency_s": 120.00699871999677, "started_at": "2026-10-21T09:30:26.002496+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-2123", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 15244, "cache_read": 109928, "completion": 9333, "prompt": 125196, "reasoning": 4976}, "upstream_digest": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "upstream_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1426", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5591005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 20.330876275002083, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "finished_at": "2026-10-21T09:33:37.300986+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1426:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.00417358199775, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 28, "overlay_image_id": "sha256:784d4d636c61b86f42753cbcb15eb3906296f932e9589b59f5946694fc90748f", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1426__opus-5__a1__20261021T093056799134Z", "solve_latency_s": 140.00417358199775, "started_at": "2026-10-21T09:30:56.799117+00:00", "success": true, "task_id": "pvlib__pvlib-python-1426", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25296, "cache_read": 360991, "completion": 8809, "prompt": 386343, "reasoning": 4171}, "upstream_digest": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893", "upstream_image_id": "sha256:c0a572b2444c19f2313bf38ed7989ab46c14b72fb5673b4a7abbcc60afe0e893"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.23195475, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "finished_at": "2026-10-21T09:35:25.636776+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-2123:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00452459700318, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-2123__haiku-4.5__a1__20261021T093245494263Z", "solve_latency_s": 160.00452459700318, "started_at": "2026-10-21T09:32:45.494244+00:00", "success": false, "task_id": "marshmallow-code__marshmallow-2123", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41305, "cache_read": 854885, "completion": 13977, "prompt": 921140, "reasoning": 0}, "upstream_digest": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f", "upstream_image_id": "sha256:9da3669fe99dede23ecf75b8f5b50198d304808f51c9ba828ac26c9eaab3163f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-807", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-807:latest", "latency_s": 41.76609422000183, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-807", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3", "upstream_image_id": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1395", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1395:latest", "latency_s": 43.451813211002445, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1395", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10", "upstream_image_id": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1832", "category": "feature_implementation", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1832:latest", "latency_s": 48.52159525999741, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1832", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f", "upstream_image_id": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1395", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.3326135, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "finished_at": "2026-10-21T09:41:05.993988+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1395:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.0135323439972, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1395__sonnet-5__a1__20261021T093825963297Z", "solve_latency_s": 160.0135323439972, "started_at": "2026-10-21T09:38:25.963276+00:00", "success": false, "task_id": "pvlib__pvlib-python-1395", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 29805, "cache_read": 773005, "completion": 10330, "prompt": 802910, "reasoning": 4834}, "upstream_digest": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10", "upstream_image_id": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-807", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2266613, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "finished_at": "2026-10-21T09:41:36.138616+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-807:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.0160952110018, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-807__haiku-4.5__a1__20261021T093825951916Z", "solve_latency_s": 190.0160952110018, "started_at": "2026-10-21T09:38:25.951895+00:00", "success": false, "task_id": "pvlib__pvlib-python-807", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37798, "cache_read": 853948, "completion": 16543, "prompt": 903050, "reasoning": 0}, "upstream_digest": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3", "upstream_image_id": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1832", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.640273, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "finished_at": "2026-10-21T09:41:36.160736+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1832:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.00935047899839, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 17, "overlay_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1832__opus-5__a1__20261021T093825970140Z", "solve_latency_s": 190.00935047899839, "started_at": "2026-10-21T09:38:25.970124+00:00", "success": false, "task_id": "pvlib__pvlib-python-1832", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 22996, "cache_read": 212456, "completion": 15606, "prompt": 235486, "reasoning": 10690}, "upstream_digest": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f", "upstream_image_id": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1395", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5689415, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "finished_at": "2026-10-21T09:43:36.270696+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1395:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.0042250330007, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 18, "overlay_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1395__opus-5__a1__20261021T094106011469Z", "solve_latency_s": 150.0042250330007, "started_at": "2026-10-21T09:41:06.011451+00:00", "success": false, "task_id": "pvlib__pvlib-python-1395", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30852, "cache_read": 311473, "completion": 8808, "prompt": 342361, "reasoning": 5738}, "upstream_digest": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10", "upstream_image_id": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-807", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2258962, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 20.844170946002123, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "finished_at": "2026-10-21T09:43:37.195917+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-807:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00476353499835, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 33, "overlay_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-807__sonnet-5__a1__20261021T094136156649Z", "solve_latency_s": 100.00476353499835, "started_at": "2026-10-21T09:41:36.156632+00:00", "success": true, "task_id": "pvlib__pvlib-python-807", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31138, "cache_read": 460446, "completion": 5583, "prompt": 491650, "reasoning": 977}, "upstream_digest": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3", "upstream_image_id": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1832", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2174364, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "finished_at": "2026-10-21T09:44:16.324434+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1832:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00643900000432, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1832__haiku-4.5__a1__20261021T094136176693Z", "solve_latency_s": 160.00643900000432, "started_at": "2026-10-21T09:41:36.176679+00:00", "success": false, "task_id": "pvlib__pvlib-python-1832", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35654, "cache_read": 878139, "completion": 14126, "prompt": 928218, "reasoning": 0}, "upstream_digest": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f", "upstream_image_id": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1395", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.19561975, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "finished_at": "2026-10-21T09:45:46.475355+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1395:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.0040684889973, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:f68e651fa4469ff35e2036af65eea1e84860ff3d720b25887521041abd3af61f", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1395__haiku-4.5__a1__20261021T094336301319Z", "solve_latency_s": 130.0040684889973, "started_at": "2026-10-21T09:43:36.301297+00:00", "success": false, "task_id": "pvlib__pvlib-python-1395", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33965, "cache_read": 826235, "completion": 10202, "prompt": 879730, "reasoning": 0}, "upstream_digest": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10", "upstream_image_id": "sha256:c7b51f4978926d54a788c54ac4d96e6c4c4e2cd2ee62736b30e91c9fd0891f10"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-807", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.63426775, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "finished_at": "2026-10-21T09:47:37.427274+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-807:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.00619818999985, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 19, "overlay_image_id": "sha256:d5a82351ce3caaa031e887ae65fefbe4e8745447e96a6ce68e88bd5b4826aeba", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-807__opus-5__a1__20261021T094337221799Z", "solve_latency_s": 240.00619818999985, "started_at": "2026-10-21T09:43:37.221778+00:00", "success": false, "task_id": "pvlib__pvlib-python-807", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 21239, "cache_read": 224168, "completion": 15570, "prompt": 245445, "reasoning": 11998}, "upstream_digest": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3", "upstream_image_id": "sha256:28abd49543ca62244c1f0b261f36892cd4024a042f3533b887ca351b24a032d3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1832", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.42225840000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "finished_at": "2026-10-21T09:47:46.546307+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1832:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.00795429199934, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:665bc8bb96aab5e75931f15715734dae3d8c2cb8faaa200bbf8f322c7ac52ff7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1832__sonnet-5__a1__20261021T094416341559Z", "solve_latency_s": 210.00795429199934, "started_at": "2026-10-21T09:44:16.341543+00:00", "success": false, "task_id": "pvlib__pvlib-python-1832", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 32502, "cache_read": 915167, "completion": 15777, "prompt": 947769, "reasoning": 6678}, "upstream_digest": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f", "upstream_image_id": "sha256:18b7e4831cb48f762575f7a6380a847d1bf9ccd3ee25fd17328b02739b2e0c9f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "1404d7b6381c443b5ad02086cf29a5f16e25395ed6aec1ef00cbb1e26d2141f4", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "env_correction", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "task_id": "pydicom__pydicom-1413", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "repository_debugging", "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "1404d7b6381c443b5ad02086cf29a5f16e25395ed6aec1ef00cbb1e26d2141f4", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "error_class": null, "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "latency_s": 37.460624332001316, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "preflight_passed", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "status": "preflight_passed", "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.18376455, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "finished_at": "2026-10-21T09:55:31.751644+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00411994499882, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1413__haiku-4.5__a1__20261021T095331618028Z", "solve_latency_s": 120.00411994499882, "started_at": "2026-10-21T09:53:31.618004+00:00", "success": false, "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33159, "cache_read": 888248, "completion": 9171, "prompt": 929043, "reasoning": 0}, "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "1404d7b6381c443b5ad02086cf29a5f16e25395ed6aec1ef00cbb1e26d2141f4", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1413__haiku-4.5__a1__20261021T095331618028Z", "task_id": "pydicom__pydicom-1413", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1805157, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.521405296996818, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "finished_at": "2026-10-21T09:57:40.454461+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.00457866500074, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 27, "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1413__sonnet-5__a1__20261021T095531778877Z", "solve_latency_s": 110.00457866500074, "started_at": "2026-10-21T09:55:31.778861+00:00", "success": true, "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17703, "cache_read": 266351, "completion": 8288, "prompt": 284108, "reasoning": 3736}, "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "1404d7b6381c443b5ad02086cf29a5f16e25395ed6aec1ef00cbb1e26d2141f4", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1413__sonnet-5__a1__20261021T095531778877Z", "task_id": "pydicom__pydicom-1413", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.60797325, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "finished_at": "2026-10-21T10:01:00.643186+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00581668200175, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 21, "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1413__opus-5__a1__20261021T095740480859Z", "solve_latency_s": 200.00581668200175, "started_at": "2026-10-21T09:57:40.480844+00:00", "success": false, "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23435, "cache_read": 260739, "completion": 13237, "prompt": 284216, "reasoning": 9997}, "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "1404d7b6381c443b5ad02086cf29a5f16e25395ed6aec1ef00cbb1e26d2141f4", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:ea2935174b546ebd183b53ca2f6feb378da154ab49947b891e1312a9e94858bb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1413__opus-5__a1__20261021T095740480859Z", "task_id": "pydicom__pydicom-1413", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; TestNumpy_NoRLEHandler and TestNumpy_RLEHandler define setup(self) to restrict pixel handlers, so both tests saw default handlers and failed", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "537a4abfca69b6009fc410550b85e90d103ace6be8e725e0e31e184fcb80c970", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_rle_pixel_data.py::TestNumpy_NoRLEHandler::test_pixel_array_raises", "pydicom/tests/test_rle_pixel_data.py::TestNumpy_RLEHandler::test_unsupported_syntax_raises"], "final_image_id": "sha256:7523d0b5c2cf51e01a7d93089bc7a1dad29ee9baf4a857b45165d304c531228e", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:7523d0b5c2cf51e01a7d93089bc7a1dad29ee9baf4a857b45165d304c531228e", "record_type": "env_correction", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "task_id": "pydicom__pydicom-1439", "upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517", "upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "repository_debugging", "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; TestNumpy_NoRLEHandler and TestNumpy_RLEHandler define setup(self) to restrict pixel handlers, so both tests saw default handlers and failed", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "537a4abfca69b6009fc410550b85e90d103ace6be8e725e0e31e184fcb80c970", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "error_class": "env", "evidence": ["pydicom/tests/test_rle_pixel_data.py::TestNumpy_NoRLEHandler::test_pixel_array_raises", "pydicom/tests/test_rle_pixel_data.py::TestNumpy_RLEHandler::test_unsupported_syntax_raises"], "final_image_id": "sha256:7523d0b5c2cf51e01a7d93089bc7a1dad29ee9baf4a857b45165d304c531228e", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest", "latency_s": 44.88570009100658, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:7523d0b5c2cf51e01a7d93089bc7a1dad29ee9baf4a857b45165d304c531228e", "record_type": "preflight_failed", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "status": "preflight_failed", "task_id": "pydicom__pydicom-1439", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517", "upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "d0d1f891f47e12454c292c5b930a50fe871f1590c1d37e77a6623052aa2c2276", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "env_correction", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "task_id": "pydicom__pydicom-1720", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "repository_debugging", "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "d0d1f891f47e12454c292c5b930a50fe871f1590c1d37e77a6623052aa2c2276", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "error_class": null, "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "latency_s": 37.70765014499921, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "preflight_passed", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "status": "preflight_passed", "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1825631, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.97216968199791, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "finished_at": "2026-10-21T10:04:36.109744+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00339116799296, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 27, "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1720__sonnet-5__a1__20261021T100246989657Z", "solve_latency_s": 90.00339116799296, "started_at": "2026-10-21T10:02:46.989632+00:00", "success": true, "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25273, "cache_read": 370713, "completion": 4513, "prompt": 396040, "reasoning": 1535}, "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "d0d1f891f47e12454c292c5b930a50fe871f1590c1d37e77a6623052aa2c2276", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1720__sonnet-5__a1__20261021T100246989657Z", "task_id": "pydicom__pydicom-1720", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.610588, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "finished_at": "2026-10-21T10:07:46.327267+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 190.00564162299997, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1720__opus-5__a1__20261021T100436136600Z", "solve_latency_s": 190.00564162299997, "started_at": "2026-10-21T10:04:36.136585+00:00", "success": false, "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23670, "cache_read": 228431, "completion": 13931, "prompt": 252133, "reasoning": 11380}, "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "d0d1f891f47e12454c292c5b930a50fe871f1590c1d37e77a6623052aa2c2276", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1720__opus-5__a1__20261021T100436136600Z", "task_id": "pydicom__pydicom-1720", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2271856, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "finished_at": "2026-10-21T10:10:26.497952+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00446149299387, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1720__haiku-4.5__a1__20261021T100746354084Z", "solve_latency_s": 160.00446149299387, "started_at": "2026-10-21T10:07:46.354060+00:00", "success": false, "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 36390, "cache_read": 1065431, "completion": 13539, "prompt": 1109281, "reasoning": 0}, "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "d0d1f891f47e12454c292c5b930a50fe871f1590c1d37e77a6623052aa2c2276", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:d7b91e7da8abba31755861e846105ac7c9b3294b0542326f9d487591686dd6dc", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1720__haiku-4.5__a1__20261021T100746354084Z", "task_id": "pydicom__pydicom-1720", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "010ca5be6575b230cb79b0c66af5d883ebd29eff5ba71c95fb9e11004bef6091", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "env_correction", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "task_id": "pydicom__pydicom-1334", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "repository_debugging", "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "010ca5be6575b230cb79b0c66af5d883ebd29eff5ba71c95fb9e11004bef6091", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "error_class": null, "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "latency_s": 38.40107596399321, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "preflight_passed", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "status": "preflight_passed", "task_id": "pydicom__pydicom-1334", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.251643, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "finished_at": "2026-10-21T10:14:50.640734+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 220.00775402400177, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1334__haiku-4.5__a1__20261021T101110392751Z", "solve_latency_s": 220.00775402400177, "started_at": "2026-10-21T10:11:10.392728+00:00", "success": false, "task_id": "pydicom__pydicom-1334", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 34044, "cache_read": 958110, "completion": 20487, "prompt": 1002996, "reasoning": 0}, "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "010ca5be6575b230cb79b0c66af5d883ebd29eff5ba71c95fb9e11004bef6091", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1334__haiku-4.5__a1__20261021T101110392751Z", "task_id": "pydicom__pydicom-1334", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.4581582, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "finished_at": "2026-10-21T10:18:20.838702+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.00515473999985, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1334__sonnet-5__a1__20261021T101450667911Z", "solve_latency_s": 210.00515473999985, "started_at": "2026-10-21T10:14:50.667896+00:00", "success": false, "task_id": "pydicom__pydicom-1334", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 46634, "cache_read": 1173116, "completion": 10675, "prompt": 1219850, "reasoning": 5179}, "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "010ca5be6575b230cb79b0c66af5d883ebd29eff5ba71c95fb9e11004bef6091", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1334__sonnet-5__a1__20261021T101450667911Z", "task_id": "pydicom__pydicom-1334", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.62260075, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "finished_at": "2026-10-21T10:21:41.975651+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00546401499741, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 19, "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1334__opus-5__a1__20261021T101820866096Z", "solve_latency_s": 200.00546401499741, "started_at": "2026-10-21T10:18:20.866080+00:00", "success": false, "task_id": "pydicom__pydicom-1334", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25559, "cache_read": 221884, "completion": 14069, "prompt": 247481, "reasoning": 11700}, "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1334-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5 and both TestBadValueRead cases failed because the test instance lacked its setup-created tag attribute", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "010ca5be6575b230cb79b0c66af5d883ebd29eff5ba71c95fb9e11004bef6091", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_default", "pydicom/tests/test_valuerep.py::TestBadValueRead::test_read_bad_value_in_VR_enforce_valid_value"], "final_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1334:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:88e0fc14a45903dae553160e5e0bfd1ebb98b021561d36e543775988b8ea5edf", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1334__opus-5__a1__20261021T101820866096Z", "task_id": "pydicom__pydicom-1334", "upstream_digest": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d", "upstream_image_id": "sha256:f66aacddba0d030bdd6618e7dca3827cf9f7a7319af8c6c86faf201f77c2dc7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit bd82f01 TestWriteFile, TestScratchWriteDateTime, TestWriteNonStandard and TestWriteUndefinedLengthPixelData create file_out/fp in setup(self), so gold and noop both failed with missing attributes", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "aff6e7fe22409578ac58d34d9a7769fadcc97c5beb0135796308aeb8a17f2ea0", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile::testRTPlan", "pydicom/tests/test_filewriter.py::TestWriteNonStandard", "pydicom/tests/test_filewriter.py::TestScratchWriteDateTime"], "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "env_correction", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "task_id": "pydicom__pydicom-1365", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "repository_debugging", "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit bd82f01 TestWriteFile, TestScratchWriteDateTime, TestWriteNonStandard and TestWriteUndefinedLengthPixelData create file_out/fp in setup(self), so gold and noop both failed with missing attributes", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "aff6e7fe22409578ac58d34d9a7769fadcc97c5beb0135796308aeb8a17f2ea0", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "error_class": null, "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile::testRTPlan", "pydicom/tests/test_filewriter.py::TestWriteNonStandard", "pydicom/tests/test_filewriter.py::TestScratchWriteDateTime"], "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "latency_s": 38.64149685099983, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "preflight_passed", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "status": "preflight_passed", "task_id": "pydicom__pydicom-1365", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6647977, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "finished_at": "2026-10-21T10:29:16.386092+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 410.00894456199603, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1365__sonnet-5__a1__20261021T102226213524Z", "solve_latency_s": 410.00894456199603, "started_at": "2026-10-21T10:22:26.213502+00:00", "success": false, "task_id": "pydicom__pydicom-1365", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 36883, "cache_read": 1145751, "completion": 34324, "prompt": 1182734, "reasoning": 29188}, "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit bd82f01 TestWriteFile, TestScratchWriteDateTime, TestWriteNonStandard and TestWriteUndefinedLengthPixelData create file_out/fp in setup(self), so gold and noop both failed with missing attributes", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "aff6e7fe22409578ac58d34d9a7769fadcc97c5beb0135796308aeb8a17f2ea0", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile::testRTPlan", "pydicom/tests/test_filewriter.py::TestWriteNonStandard", "pydicom/tests/test_filewriter.py::TestScratchWriteDateTime"], "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1365__sonnet-5__a1__20261021T102226213524Z", "task_id": "pydicom__pydicom-1365", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5869015, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "finished_at": "2026-10-21T10:31:46.577828+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00461515400093, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 17, "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1365__opus-5__a1__20261021T102916415027Z", "solve_latency_s": 150.00461515400093, "started_at": "2026-10-21T10:29:16.415012+00:00", "success": false, "task_id": "pydicom__pydicom-1365", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 27974, "cache_read": 277588, "completion": 10924, "prompt": 305596, "reasoning": 7528}, "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit bd82f01 TestWriteFile, TestScratchWriteDateTime, TestWriteNonStandard and TestWriteUndefinedLengthPixelData create file_out/fp in setup(self), so gold and noop both failed with missing attributes", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "aff6e7fe22409578ac58d34d9a7769fadcc97c5beb0135796308aeb8a17f2ea0", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile::testRTPlan", "pydicom/tests/test_filewriter.py::TestWriteNonStandard", "pydicom/tests/test_filewriter.py::TestScratchWriteDateTime"], "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1365__opus-5__a1__20261021T102916415027Z", "task_id": "pydicom__pydicom-1365", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.19556695, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "finished_at": "2026-10-21T10:34:06.753258+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 140.00554929800273, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1365__haiku-4.5__a1__20261021T103146606144Z", "solve_latency_s": 140.00554929800273, "started_at": "2026-10-21T10:31:46.606123+00:00", "success": false, "task_id": "pydicom__pydicom-1365", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33263, "cache_read": 820312, "completion": 12690, "prompt": 862082, "reasoning": 0}, "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1365-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit bd82f01 TestWriteFile, TestScratchWriteDateTime, TestWriteNonStandard and TestWriteUndefinedLengthPixelData create file_out/fp in setup(self), so gold and noop both failed with missing attributes", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "aff6e7fe22409578ac58d34d9a7769fadcc97c5beb0135796308aeb8a17f2ea0", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile::testRTPlan", "pydicom/tests/test_filewriter.py::TestWriteNonStandard", "pydicom/tests/test_filewriter.py::TestScratchWriteDateTime"], "final_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1365:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:2e86856668ebe0aa73172c9e53545f51976e046e0a33218192d9033065568ab7", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1365__haiku-4.5__a1__20261021T103146606144Z", "task_id": "pydicom__pydicom-1365", "upstream_digest": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2", "upstream_image_id": "sha256:c47b4a1a8f24f3a50f71fbe5c2be79485957f0dda8524a56bd7875d571008cd2"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit 00c2484 TestWriteFile, TestWriteNonStandard, TestReadDataElement, TestIncorrectVR and TestDeferredRead build fixtures in setup(self), and TestReader/TestIncorrectVR teardown(self) resets config.enforce_valid_values, so skipped teardowns also leave strict-mode state that makes later charset tests raise instead of warn. Gold revalidation on the corrected image decides whether every failure is covered", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "ea3e01449e3e4ef3f778007e1af513607660fe3046901c2e7dac2869dc74808f", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile", "pydicom/tests/test_filereader.py::TestReadDataElement", "pydicom/tests/test_filereader.py::TestDeferredRead::test_file_exists", "pydicom/tests/test_filereader.py::TestReader::test_long_specific_char_set"], "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "env_correction", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "task_id": "pydicom__pydicom-1048", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "repository_debugging", "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit 00c2484 TestWriteFile, TestWriteNonStandard, TestReadDataElement, TestIncorrectVR and TestDeferredRead build fixtures in setup(self), and TestReader/TestIncorrectVR teardown(self) resets config.enforce_valid_values, so skipped teardowns also leave strict-mode state that makes later charset tests raise instead of warn. Gold revalidation on the corrected image decides whether every failure is covered", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "ea3e01449e3e4ef3f778007e1af513607660fe3046901c2e7dac2869dc74808f", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "error_class": null, "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile", "pydicom/tests/test_filereader.py::TestReadDataElement", "pydicom/tests/test_filereader.py::TestDeferredRead::test_file_exists", "pydicom/tests/test_filereader.py::TestReader::test_long_specific_char_set"], "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "latency_s": 37.73353066199343, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "preflight_passed", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "status": "preflight_passed", "task_id": "pydicom__pydicom-1048", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.13776, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 19.02567490000365, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "finished_at": "2026-10-21T10:36:37.033171+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 70.00294118400052, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 28, "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1048__sonnet-5__a1__20261021T103507858137Z", "solve_latency_s": 70.00294118400052, "started_at": "2026-10-21T10:35:07.858114+00:00", "success": false, "task_id": "pydicom__pydicom-1048", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17180, "cache_read": 243440, "completion": 4601, "prompt": 260676, "reasoning": 751}, "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit 00c2484 TestWriteFile, TestWriteNonStandard, TestReadDataElement, TestIncorrectVR and TestDeferredRead build fixtures in setup(self), and TestReader/TestIncorrectVR teardown(self) resets config.enforce_valid_values, so skipped teardowns also leave strict-mode state that makes later charset tests raise instead of warn. Gold revalidation on the corrected image decides whether every failure is covered", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "ea3e01449e3e4ef3f778007e1af513607660fe3046901c2e7dac2869dc74808f", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile", "pydicom/tests/test_filereader.py::TestReadDataElement", "pydicom/tests/test_filereader.py::TestDeferredRead::test_file_exists", "pydicom/tests/test_filereader.py::TestReader::test_long_specific_char_set"], "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1048__sonnet-5__a1__20261021T103507858137Z", "task_id": "pydicom__pydicom-1048", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.57671375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "finished_at": "2026-10-21T10:39:07.086083+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.0065340190049, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 19, "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1048__opus-5__a1__20261021T103637063697Z", "solve_latency_s": 150.0065340190049, "started_at": "2026-10-21T10:36:37.063682+00:00", "success": false, "task_id": "pydicom__pydicom-1048", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28567, "cache_read": 338160, "completion": 9156, "prompt": 366765, "reasoning": 3881}, "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit 00c2484 TestWriteFile, TestWriteNonStandard, TestReadDataElement, TestIncorrectVR and TestDeferredRead build fixtures in setup(self), and TestReader/TestIncorrectVR teardown(self) resets config.enforce_valid_values, so skipped teardowns also leave strict-mode state that makes later charset tests raise instead of warn. Gold revalidation on the corrected image decides whether every failure is covered", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "ea3e01449e3e4ef3f778007e1af513607660fe3046901c2e7dac2869dc74808f", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile", "pydicom/tests/test_filereader.py::TestReadDataElement", "pydicom/tests/test_filereader.py::TestDeferredRead::test_file_exists", "pydicom/tests/test_filereader.py::TestReader::test_long_specific_char_set"], "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1048__opus-5__a1__20261021T103637063697Z", "task_id": "pydicom__pydicom-1048", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.07936725, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.622337238004548, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "finished_at": "2026-10-21T10:40:35.808181+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 70.00288204699609, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 28, "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1048__haiku-4.5__a1__20261021T103907114793Z", "solve_latency_s": 70.00288204699609, "started_at": "2026-10-21T10:39:07.114777+00:00", "success": false, "task_id": "pydicom__pydicom-1048", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 16755, "cache_read": 205555, "completion": 4848, "prompt": 235938, "reasoning": 0}, "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1048-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which no longer calls nose-style setup/teardown; at test-file commit 00c2484 TestWriteFile, TestWriteNonStandard, TestReadDataElement, TestIncorrectVR and TestDeferredRead build fixtures in setup(self), and TestReader/TestIncorrectVR teardown(self) resets config.enforce_valid_values, so skipped teardowns also leave strict-mode state that makes later charset tests raise instead of warn. Gold revalidation on the corrected image decides whether every failure is covered", "corrections_file_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "cost_usd": 0.0, "dockerfile_sha256": "ea3e01449e3e4ef3f778007e1af513607660fe3046901c2e7dac2869dc74808f", "env_correction_sha256": "3354b4543cd34bec6136c12fe43eadd209e566971886df5de086732cf85e0719", "evidence": ["pydicom/tests/test_filewriter.py::TestWriteFile", "pydicom/tests/test_filereader.py::TestReadDataElement", "pydicom/tests/test_filereader.py::TestDeferredRead::test_file_exists", "pydicom/tests/test_filereader.py::TestReader::test_long_specific_char_set"], "final_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1048:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:47a5d7a3810bc9dfbf5e8f23a8b9fc812c0b1f7873c06e89a1ed49d0689014e4", "record_type": "env_correction_attempt_link", "repair_script_sha256": "5742e7223c9579f24d77080f58ea1140734f82bdd23b6ad9458fe66104e81a59", "run_id": "pydicom__pydicom-1048__haiku-4.5__a1__20261021T103907114793Z", "task_id": "pydicom__pydicom-1048", "upstream_digest": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f", "upstream_image_id": "sha256:927f68ae738299106c329fb7d41f4dfae89f65093a890f467f871319b75cc61f"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-432-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "cause": "Gold and noop preflights both failed at collection with ImportError: libXrender.so.1: cannot open shared object file, so no FAIL_TO_PASS test could run; the image lacks the X11 render library VTK loads at import", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b62d6429e68a1e8f92a8d8c2cdbdedae5d0b43571d4eb724c5e094b5acda2c32", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_plotting.py::test_invalid_color", "tests/test_plotting.py::test_invalid_font"], "final_image_id": "sha256:4e818cdda198e7fac1aa95e7a29c9446f398d918b97ce5cc651ea571a141d0e9", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:4e818cdda198e7fac1aa95e7a29c9446f398d918b97ce5cc651ea571a141d0e9", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pyvista__pyvista-432", "upstream_digest": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21", "upstream_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-432-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "category": "feature_implementation", "cause": "Gold and noop preflights both failed at collection with ImportError: libXrender.so.1: cannot open shared object file, so no FAIL_TO_PASS test could run; the image lacks the X11 render library VTK loads at import", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b62d6429e68a1e8f92a8d8c2cdbdedae5d0b43571d4eb724c5e094b5acda2c32", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": "env", "evidence": ["tests/test_plotting.py::test_invalid_color", "tests/test_plotting.py::test_invalid_font"], "final_image_id": "sha256:4e818cdda198e7fac1aa95e7a29c9446f398d918b97ce5cc651ea571a141d0e9", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-432:latest", "latency_s": 38.10395569400134, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:4e818cdda198e7fac1aa95e7a29c9446f398d918b97ce5cc651ea571a141d0e9", "record_type": "preflight_failed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_failed", "task_id": "pyvista__pyvista-432", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21", "upstream_image_id": "sha256:aeb69732fbbe76f5f308395122531837fd109284759ef8f08aa2f8a42be00d21"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-24. Pre-solve diagnostic in a throwaway container: 6 failed -> 6 passed (base checkout, failing PASS_TO_PASS subset)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "bf696439e5a674628cac436641c7d92db84894d4f5d8ae6e5fa2bf5f7d2291ca", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_grid.py::test_volume", "tests/test_grid.py::test_init_from_structured", "tests/test_grid.py::test_init_from_unstructured", "tests/test_grid.py::test_surface_indices", "tests/test_grid.py::test_extract_feature_edges"], "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pyvista__pyvista-4329", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "category": "repository_debugging", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-24. Pre-solve diagnostic in a throwaway container: 6 failed -> 6 passed (base checkout, failing PASS_TO_PASS subset)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "bf696439e5a674628cac436641c7d92db84894d4f5d8ae6e5fa2bf5f7d2291ca", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": null, "evidence": ["tests/test_grid.py::test_volume", "tests/test_grid.py::test_init_from_structured", "tests/test_grid.py::test_init_from_unstructured", "tests/test_grid.py::test_surface_indices", "tests/test_grid.py::test_extract_feature_edges"], "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "latency_s": 47.33943082299811, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "preflight_passed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_passed", "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1743937, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "finished_at": "2026-10-21T11:09:12.499911+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00448943200172, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pyvista__pyvista-4329__haiku-4.5__a1__20261021T110702350056Z", "solve_latency_s": 130.00448943200172, "started_at": "2026-10-21T11:07:02.350030+00:00", "success": false, "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 28692, "cache_read": 697307, "completion": 10624, "prompt": 741677, "reasoning": 0}, "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-24. Pre-solve diagnostic in a throwaway container: 6 failed -> 6 passed (base checkout, failing PASS_TO_PASS subset)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "bf696439e5a674628cac436641c7d92db84894d4f5d8ae6e5fa2bf5f7d2291ca", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_grid.py::test_volume", "tests/test_grid.py::test_init_from_structured", "tests/test_grid.py::test_init_from_unstructured", "tests/test_grid.py::test_surface_indices", "tests/test_grid.py::test_extract_feature_edges"], "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4329__haiku-4.5__a1__20261021T110702350056Z", "task_id": "pyvista__pyvista-4329", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5416746, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "finished_at": "2026-10-21T11:14:22.727224+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 310.00993738700345, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pyvista__pyvista-4329__sonnet-5__a1__20261021T110912554070Z", "solve_latency_s": 310.00993738700345, "started_at": "2026-10-21T11:09:12.554051+00:00", "success": false, "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 39440, "cache_read": 1030523, "completion": 23677, "prompt": 1070063, "reasoning": 12811}, "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-24. Pre-solve diagnostic in a throwaway container: 6 failed -> 6 passed (base checkout, failing PASS_TO_PASS subset)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "bf696439e5a674628cac436641c7d92db84894d4f5d8ae6e5fa2bf5f7d2291ca", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_grid.py::test_volume", "tests/test_grid.py::test_init_from_structured", "tests/test_grid.py::test_init_from_unstructured", "tests/test_grid.py::test_surface_indices", "tests/test_grid.py::test_extract_feature_edges"], "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4329__sonnet-5__a1__20261021T110912554070Z", "task_id": "pyvista__pyvista-4329", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.60623475, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "finished_at": "2026-10-21T11:17:42.896739+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00774055499642, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 20, "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pyvista__pyvista-4329__opus-5__a1__20261021T111422757066Z", "solve_latency_s": 200.00774055499642, "started_at": "2026-10-21T11:14:22.757050+00:00", "success": false, "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24583, "cache_read": 285582, "completion": 12384, "prompt": 310205, "reasoning": 9279}, "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4329-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-24. Pre-solve diagnostic in a throwaway container: 6 failed -> 6 passed (base checkout, failing PASS_TO_PASS subset)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "bf696439e5a674628cac436641c7d92db84894d4f5d8ae6e5fa2bf5f7d2291ca", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_grid.py::test_volume", "tests/test_grid.py::test_init_from_structured", "tests/test_grid.py::test_init_from_unstructured", "tests/test_grid.py::test_surface_indices", "tests/test_grid.py::test_extract_feature_edges"], "final_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:f68bda9f172460910ce453e75895a73d543633c3e0cce14fea72c80b677947ef", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4329__opus-5__a1__20261021T111422757066Z", "task_id": "pyvista__pyvista-4329", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-05-12. Pre-solve diagnostic in a throwaway container: 5 failed, 55 errors -> 60 passed (gold applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "053bbfb4b7706d950e9720d3fe3df22755e2dee0262f9d60127d2f757bc56d5d", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_polydata.py::test_merge", "tests/test_polydata.py::test_init_from_pdata", "tests/test_polydata.py::test_init_from_arrays", "tests/test_polydata.py::test_init_as_points", "tests/test_polydata.py::test_init_as_points_from_list"], "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pyvista__pyvista-4414", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "category": "repository_debugging", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-05-12. Pre-solve diagnostic in a throwaway container: 5 failed, 55 errors -> 60 passed (gold applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "053bbfb4b7706d950e9720d3fe3df22755e2dee0262f9d60127d2f757bc56d5d", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": null, "evidence": ["tests/test_polydata.py::test_merge", "tests/test_polydata.py::test_init_from_pdata", "tests/test_polydata.py::test_init_from_arrays", "tests/test_polydata.py::test_init_as_points", "tests/test_polydata.py::test_init_as_points_from_list"], "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "latency_s": 45.06382991999999, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "preflight_passed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_passed", "task_id": "pyvista__pyvista-4414", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2056859, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "finished_at": "2026-10-21T11:21:35.074100+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00426578099723, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pyvista__pyvista-4414__haiku-4.5__a1__20261021T111924935539Z", "solve_latency_s": 130.00426578099723, "started_at": "2026-10-21T11:19:24.935516+00:00", "success": false, "task_id": "pyvista__pyvista-4414", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35812, "cache_read": 1050529, "completion": 9977, "prompt": 1092324, "reasoning": 0}, "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-05-12. Pre-solve diagnostic in a throwaway container: 5 failed, 55 errors -> 60 passed (gold applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "053bbfb4b7706d950e9720d3fe3df22755e2dee0262f9d60127d2f757bc56d5d", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_polydata.py::test_merge", "tests/test_polydata.py::test_init_from_pdata", "tests/test_polydata.py::test_init_from_arrays", "tests/test_polydata.py::test_init_as_points", "tests/test_polydata.py::test_init_as_points_from_list"], "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4414__haiku-4.5__a1__20261021T111924935539Z", "task_id": "pyvista__pyvista-4414", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5699537, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "finished_at": "2026-10-21T11:26:05.256484+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 270.0074872670011, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pyvista__pyvista-4414__sonnet-5__a1__20261021T112135108522Z", "solve_latency_s": 270.0074872670011, "started_at": "2026-10-21T11:21:35.108505+00:00", "success": false, "task_id": "pyvista__pyvista-4414", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 44447, "cache_read": 1291731, "completion": 20029, "prompt": 1336278, "reasoning": 10334}, "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-05-12. Pre-solve diagnostic in a throwaway container: 5 failed, 55 errors -> 60 passed (gold applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "053bbfb4b7706d950e9720d3fe3df22755e2dee0262f9d60127d2f757bc56d5d", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_polydata.py::test_merge", "tests/test_polydata.py::test_init_from_pdata", "tests/test_polydata.py::test_init_from_arrays", "tests/test_polydata.py::test_init_as_points", "tests/test_polydata.py::test_init_as_points_from_list"], "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4414__sonnet-5__a1__20261021T112135108522Z", "task_id": "pyvista__pyvista-4414", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.611785, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "finished_at": "2026-10-21T11:29:35.460237+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.00579061399912, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 25, "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pyvista__pyvista-4414__opus-5__a1__20261021T112605286493Z", "solve_latency_s": 210.00579061399912, "started_at": "2026-10-21T11:26:05.286476+00:00", "success": false, "task_id": "pyvista__pyvista-4414", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25078, "cache_read": 358595, "completion": 11020, "prompt": 383723, "reasoning": 6012}, "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4414-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-05-12. Pre-solve diagnostic in a throwaway container: 5 failed, 55 errors -> 60 passed (gold applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "053bbfb4b7706d950e9720d3fe3df22755e2dee0262f9d60127d2f757bc56d5d", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/test_polydata.py::test_merge", "tests/test_polydata.py::test_init_from_pdata", "tests/test_polydata.py::test_init_from_arrays", "tests/test_polydata.py::test_init_as_points", "tests/test_polydata.py::test_init_as_points_from_list"], "final_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4414:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:c07867e9078e09099e461f25fadfa6d38b49146f3d3405448f7a4d1b686befc8", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4414__opus-5__a1__20261021T112605286493Z", "task_id": "pyvista__pyvista-4414", "upstream_digest": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92", "upstream_image_id": "sha256:b6a3ebf210abaaa95e87e1ef636c9f0ab6fd9ade72bc9437eafa9347fa1acd92"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.2 is the latest cp39 release on PyPI before the task created_at 2022-12-02. Pre-solve diagnostic in a throwaway container: 4 failed, 27 errors -> 31 passed (gold and test patch applied, failing FAIL_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "f9814d6dffb7f2e8ef6115c29819e68f8cfb6b017c79f10fbfd51945917135f6", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/mappers/test_mapper.py::test_init", "tests/mappers/test_mapper.py::test_scalar_range", "tests/mappers/test_mapper.py::test_bounds", "tests/mappers/test_mapper.py::test_lookup_table", "tests/mappers/test_mapper.py::test_interpolate_before_map"], "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pyvista__pyvista-3675", "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "category": "feature_implementation", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.2 is the latest cp39 release on PyPI before the task created_at 2022-12-02. Pre-solve diagnostic in a throwaway container: 4 failed, 27 errors -> 31 passed (gold and test patch applied, failing FAIL_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "f9814d6dffb7f2e8ef6115c29819e68f8cfb6b017c79f10fbfd51945917135f6", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": null, "evidence": ["tests/mappers/test_mapper.py::test_init", "tests/mappers/test_mapper.py::test_scalar_range", "tests/mappers/test_mapper.py::test_bounds", "tests/mappers/test_mapper.py::test_lookup_table", "tests/mappers/test_mapper.py::test_interpolate_before_map"], "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "latency_s": 45.164187762005895, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "preflight_passed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_passed", "task_id": "pyvista__pyvista-3675", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.25650745, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "finished_at": "2026-10-21T11:34:38.300181+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00530160799826, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pyvista__pyvista-3675__haiku-4.5__a1__20261021T113118150850Z", "solve_latency_s": 200.00530160799826, "started_at": "2026-10-21T11:31:18.150828+00:00", "success": false, "task_id": "pyvista__pyvista-3675", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35567, "cache_read": 1025047, "completion": 20383, "prompt": 1068243, "reasoning": 0}, "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.2 is the latest cp39 release on PyPI before the task created_at 2022-12-02. Pre-solve diagnostic in a throwaway container: 4 failed, 27 errors -> 31 passed (gold and test patch applied, failing FAIL_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "f9814d6dffb7f2e8ef6115c29819e68f8cfb6b017c79f10fbfd51945917135f6", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/mappers/test_mapper.py::test_init", "tests/mappers/test_mapper.py::test_scalar_range", "tests/mappers/test_mapper.py::test_bounds", "tests/mappers/test_mapper.py::test_lookup_table", "tests/mappers/test_mapper.py::test_interpolate_before_map"], "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-3675__haiku-4.5__a1__20261021T113118150850Z", "task_id": "pyvista__pyvista-3675", "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.5362429, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "finished_at": "2026-10-21T11:37:38.489789+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00570062099723, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pyvista__pyvista-3675__sonnet-5__a1__20261021T113438330570Z", "solve_latency_s": 180.00570062099723, "started_at": "2026-10-21T11:34:38.330555+00:00", "success": false, "task_id": "pyvista__pyvista-3675", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 52553, "cache_read": 1320802, "completion": 14050, "prompt": 1373455, "reasoning": 2937}, "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.2 is the latest cp39 release on PyPI before the task created_at 2022-12-02. Pre-solve diagnostic in a throwaway container: 4 failed, 27 errors -> 31 passed (gold and test patch applied, failing FAIL_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "f9814d6dffb7f2e8ef6115c29819e68f8cfb6b017c79f10fbfd51945917135f6", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/mappers/test_mapper.py::test_init", "tests/mappers/test_mapper.py::test_scalar_range", "tests/mappers/test_mapper.py::test_bounds", "tests/mappers/test_mapper.py::test_lookup_table", "tests/mappers/test_mapper.py::test_interpolate_before_map"], "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-3675__sonnet-5__a1__20261021T113438330570Z", "task_id": "pyvista__pyvista-3675", "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.572586, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "finished_at": "2026-10-21T11:39:48.685138+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00421959399682, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 18, "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pyvista__pyvista-3675__opus-5__a1__20261021T113738527631Z", "solve_latency_s": 130.00421959399682, "started_at": "2026-10-21T11:37:38.527612+00:00", "success": false, "task_id": "pyvista__pyvista-3675", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31674, "cache_read": 320137, "completion": 8575, "prompt": 351847, "reasoning": 5977}, "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-3675-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.2 is the latest cp39 release on PyPI before the task created_at 2022-12-02. Pre-solve diagnostic in a throwaway container: 4 failed, 27 errors -> 31 passed (gold and test patch applied, failing FAIL_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "f9814d6dffb7f2e8ef6115c29819e68f8cfb6b017c79f10fbfd51945917135f6", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/mappers/test_mapper.py::test_init", "tests/mappers/test_mapper.py::test_scalar_range", "tests/mappers/test_mapper.py::test_bounds", "tests/mappers/test_mapper.py::test_lookup_table", "tests/mappers/test_mapper.py::test_interpolate_before_map"], "final_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-3675:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:49fdd113eb10111eff0eb91033fe0845430c69e80c003442b6931d32d90aafcb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-3675__opus-5__a1__20261021T113738527631Z", "task_id": "pyvista__pyvista-3675", "upstream_digest": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7", "upstream_image_id": "sha256:6aea7ac7e7fd73861304d89266e30e2d9d70cf955fb9cb84177f5d001bcd34a7"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-07-11. Pre-solve diagnostic in a throwaway container: 48 failed, 12 passed -> 60 passed (gold and test patch applied, first 60 FAIL_TO_PASS+PASS_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "66c2db8a1a36d1bbb6210abf7c6efaf6f219425d7e5775710a2042298d8213df", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_dataset_filters.py::test_sample", "tests/filters/test_dataset_filters.py::test_probe[None-True-True]", "tests/filters/test_dataset_filters.py::test_probe[None-True-False]", "tests/filters/test_dataset_filters.py::test_probe[None-False-True]", "tests/filters/test_dataset_filters.py::test_probe[None-False-False]"], "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pyvista__pyvista-4648", "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "category": "feature_implementation", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-07-11. Pre-solve diagnostic in a throwaway container: 48 failed, 12 passed -> 60 passed (gold and test patch applied, first 60 FAIL_TO_PASS+PASS_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "66c2db8a1a36d1bbb6210abf7c6efaf6f219425d7e5775710a2042298d8213df", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": null, "evidence": ["tests/filters/test_dataset_filters.py::test_sample", "tests/filters/test_dataset_filters.py::test_probe[None-True-True]", "tests/filters/test_dataset_filters.py::test_probe[None-True-False]", "tests/filters/test_dataset_filters.py::test_probe[None-False-True]", "tests/filters/test_dataset_filters.py::test_probe[None-False-False]"], "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "latency_s": 71.9319600279996, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "preflight_passed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_passed", "task_id": "pyvista__pyvista-4648", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.3569149, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 35.67319734199555, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "finished_at": "2026-10-21T11:45:01.186311+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00472862999595, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 44, "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pyvista__pyvista-4648__sonnet-5__a1__20261021T114145313859Z", "solve_latency_s": 160.00472862999595, "started_at": "2026-10-21T11:41:45.313837+00:00", "success": true, "task_id": "pyvista__pyvista-4648", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 41749, "cache_read": 840882, "completion": 8419, "prompt": 882719, "reasoning": 3418}, "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-07-11. Pre-solve diagnostic in a throwaway container: 48 failed, 12 passed -> 60 passed (gold and test patch applied, first 60 FAIL_TO_PASS+PASS_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "66c2db8a1a36d1bbb6210abf7c6efaf6f219425d7e5775710a2042298d8213df", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_dataset_filters.py::test_sample", "tests/filters/test_dataset_filters.py::test_probe[None-True-True]", "tests/filters/test_dataset_filters.py::test_probe[None-True-False]", "tests/filters/test_dataset_filters.py::test_probe[None-False-True]", "tests/filters/test_dataset_filters.py::test_probe[None-False-False]"], "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4648__sonnet-5__a1__20261021T114145313859Z", "task_id": "pyvista__pyvista-4648", "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.6261405, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "finished_at": "2026-10-21T11:48:31.388919+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 210.00876332999906, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 17, "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pyvista__pyvista-4648__opus-5__a1__20261021T114501220816Z", "solve_latency_s": 210.00876332999906, "started_at": "2026-10-21T11:45:01.220799+00:00", "success": false, "task_id": "pyvista__pyvista-4648", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 20894, "cache_read": 206516, "completion": 15685, "prompt": 227444, "reasoning": 12999}, "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-07-11. Pre-solve diagnostic in a throwaway container: 48 failed, 12 passed -> 60 passed (gold and test patch applied, first 60 FAIL_TO_PASS+PASS_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "66c2db8a1a36d1bbb6210abf7c6efaf6f219425d7e5775710a2042298d8213df", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_dataset_filters.py::test_sample", "tests/filters/test_dataset_filters.py::test_probe[None-True-True]", "tests/filters/test_dataset_filters.py::test_probe[None-True-False]", "tests/filters/test_dataset_filters.py::test_probe[None-False-True]", "tests/filters/test_dataset_filters.py::test_probe[None-False-False]"], "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4648__opus-5__a1__20261021T114501220816Z", "task_id": "pyvista__pyvista-4648", "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.25263175, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "finished_at": "2026-10-21T11:52:21.574203+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.0074595029946, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pyvista__pyvista-4648__haiku-4.5__a1__20261021T114831425281Z", "solve_latency_s": 230.0074595029946, "started_at": "2026-10-21T11:48:31.425264+00:00", "success": false, "task_id": "pyvista__pyvista-4648", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37577, "cache_read": 992785, "completion": 18791, "prompt": 1042789, "reasoning": 0}, "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4648-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-07-11. Pre-solve diagnostic in a throwaway container: 48 failed, 12 passed -> 60 passed (gold and test patch applied, first 60 FAIL_TO_PASS+PASS_TO_PASS)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "66c2db8a1a36d1bbb6210abf7c6efaf6f219425d7e5775710a2042298d8213df", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_dataset_filters.py::test_sample", "tests/filters/test_dataset_filters.py::test_probe[None-True-True]", "tests/filters/test_dataset_filters.py::test_probe[None-True-False]", "tests/filters/test_dataset_filters.py::test_probe[None-False-True]", "tests/filters/test_dataset_filters.py::test_probe[None-False-False]"], "final_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4648:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:0fcce65dafd0da228f1cc87ba33548f720397568cc49285339909f28ba59c4a2", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4648__haiku-4.5__a1__20261021T114831425281Z", "task_id": "pyvista__pyvista-4648", "upstream_digest": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455", "upstream_image_id": "sha256:ba323e45c4e076da80413910ffbef577f7ee9183ecece03b38abd683f78da455"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-20. Pre-solve diagnostic in a throwaway container: 7 failed -> 7 passed (gold and test patch applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b8a487d5907dab434ee835b929d6a67377932d3f1c5e3651fe6a6182d42e10ff", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_ids", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_data", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[5]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[6]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[12]"], "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pyvista__pyvista-4311", "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "category": "feature_implementation", "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-20. Pre-solve diagnostic in a throwaway container: 7 failed -> 7 passed (gold and test patch applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b8a487d5907dab434ee835b929d6a67377932d3f1c5e3651fe6a6182d42e10ff", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": null, "evidence": ["tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_ids", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_data", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[5]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[6]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[12]"], "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "latency_s": 45.9595374940036, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "preflight_passed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_passed", "task_id": "pyvista__pyvista-4311", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.64461725, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "finished_at": "2026-10-21T12:10:04.153749+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00756113300304, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 13, "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pyvista__pyvista-4311__opus-5__a1__20261021T120614007472Z", "solve_latency_s": 230.00756113300304, "started_at": "2026-10-21T12:06:14.007448+00:00", "success": false, "task_id": "pyvista__pyvista-4311", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 20579, "cache_read": 127637, "completion": 18082, "prompt": 148242, "reasoning": 13280}, "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-20. Pre-solve diagnostic in a throwaway container: 7 failed -> 7 passed (gold and test patch applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b8a487d5907dab434ee835b929d6a67377932d3f1c5e3651fe6a6182d42e10ff", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_ids", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_data", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[5]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[6]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[12]"], "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4311__opus-5__a1__20261021T120614007472Z", "task_id": "pyvista__pyvista-4311", "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.21298115, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.682548650998797, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "finished_at": "2026-10-21T12:13:46.010441+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00572107599874, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 40, "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pyvista__pyvista-4311__haiku-4.5__a1__20261021T121004186790Z", "solve_latency_s": 200.00572107599874, "started_at": "2026-10-21T12:10:04.186774+00:00", "success": false, "task_id": "pyvista__pyvista-4311", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35047, "cache_read": 698514, "completion": 18362, "prompt": 741072, "reasoning": 0}, "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-20. Pre-solve diagnostic in a throwaway container: 7 failed -> 7 passed (gold and test patch applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b8a487d5907dab434ee835b929d6a67377932d3f1c5e3651fe6a6182d42e10ff", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_ids", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_data", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[5]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[6]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[12]"], "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4311__haiku-4.5__a1__20261021T121004186790Z", "task_id": "pyvista__pyvista-4311", "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.2938335, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 24.315481279001688, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "finished_at": "2026-10-21T12:16:40.564543+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.0042605149938, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 38, "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pyvista__pyvista-4311__sonnet-5__a1__20261021T121346042836Z", "solve_latency_s": 150.0042605149938, "started_at": "2026-10-21T12:13:46.042810+00:00", "success": false, "task_id": "pyvista__pyvista-4311", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 29411, "cache_read": 537970, "completion": 11256, "prompt": 567457, "reasoning": 1930}, "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pyvista__pyvista-4311-envfix1", "applies_on_top_of": "none (no pyvista repo overlay; pinned upstream image)", "attempt": 1, "cause": "The image resolved unpinned vtk to 9.4.2 (released 2025-03-28), which this pyvista version does not support. Gold and noop failed identically with VTK API errors. vtk 9.2.6 is the latest cp39 release on PyPI before the task created_at 2023-04-20. Pre-solve diagnostic in a throwaway container: 7 failed -> 7 passed (gold and test patch applied, failing tests)", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "b8a487d5907dab434ee835b929d6a67377932d3f1c5e3651fe6a6182d42e10ff", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_ids", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral_pass_cell_data", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[5]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[6]", "tests/filters/test_rectilinear_grid.py::test_to_tetrahedral[12]"], "final_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4311:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:95c6f91ed03697ee0eb1572caba6b09847f15a3dc094dbbd6f7b75c7db5ca6cb", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pyvista__pyvista-4311__sonnet-5__a1__20261021T121346042836Z", "task_id": "pyvista__pyvista-4311", "upstream_digest": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453", "upstream_image_id": "sha256:b7e495f378ad4dd9bbfe2aa8e49b5a0592dbdb386a5a8eff419a4fc219b4f453"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "cause": "The gold preflight ran pytest 8.3.5, which ignores nose-style setup/teardown. At base commit 674da68 TestFileSet_Modify and TestFileSet_Copy patch FileSet.__len__ to report 2176782336+ instances and restore it only in teardown(self), so the patch leaks and later file-set writes raise NotImplementedError", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "07708bb0743612bb86f89b92d3baf58f716383c09f60902ef9aef9eb22267d70", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["pydicom/tests/test_fileset.py::TestFileSet_Modify", "pydicom/tests/test_fileset.py::TestFileSet_Copy"], "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "env_correction", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "task_id": "pydicom__pydicom-1428", "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "category": "feature_implementation", "cause": "The gold preflight ran pytest 8.3.5, which ignores nose-style setup/teardown. At base commit 674da68 TestFileSet_Modify and TestFileSet_Copy patch FileSet.__len__ to report 2176782336+ instances and restore it only in teardown(self), so the patch leaks and later file-set writes raise NotImplementedError", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "07708bb0743612bb86f89b92d3baf58f716383c09f60902ef9aef9eb22267d70", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "error_class": null, "evidence": ["pydicom/tests/test_fileset.py::TestFileSet_Modify", "pydicom/tests/test_fileset.py::TestFileSet_Copy"], "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "latency_s": 43.78901798700099, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "preflight_passed", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "status": "preflight_passed", "task_id": "pydicom__pydicom-1428", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.18170235, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.667963424995833, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "finished_at": "2026-10-21T12:20:30.742721+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00689898100245, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 42, "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1428__haiku-4.5__a1__20261021T121738897568Z", "solve_latency_s": 150.00689898100245, "started_at": "2026-10-21T12:17:38.897542+00:00", "success": false, "task_id": "pydicom__pydicom-1428", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 34681, "cache_read": 693221, "completion": 12970, "prompt": 732081, "reasoning": 0}, "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which ignores nose-style setup/teardown. At base commit 674da68 TestFileSet_Modify and TestFileSet_Copy patch FileSet.__len__ to report 2176782336+ instances and restore it only in teardown(self), so the patch leaks and later file-set writes raise NotImplementedError", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "07708bb0743612bb86f89b92d3baf58f716383c09f60902ef9aef9eb22267d70", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["pydicom/tests/test_fileset.py::TestFileSet_Modify", "pydicom/tests/test_fileset.py::TestFileSet_Copy"], "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "model_alias": "haiku-4.5", "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pydicom__pydicom-1428__haiku-4.5__a1__20261021T121738897568Z", "task_id": "pydicom__pydicom-1428", "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.0723054, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 21.637809288993594, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "finished_at": "2026-10-21T12:21:52.457361+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 60.00730703199952, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 14, "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1428__sonnet-5__a1__20261021T122030787782Z", "solve_latency_s": 60.00730703199952, "started_at": "2026-10-21T12:20:30.787765+00:00", "success": false, "task_id": "pydicom__pydicom-1428", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 10784, "cache_read": 77147, "completion": 2986, "prompt": 87959, "reasoning": 110}, "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which ignores nose-style setup/teardown. At base commit 674da68 TestFileSet_Modify and TestFileSet_Copy patch FileSet.__len__ to report 2176782336+ instances and restore it only in teardown(self), so the patch leaks and later file-set writes raise NotImplementedError", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "07708bb0743612bb86f89b92d3baf58f716383c09f60902ef9aef9eb22267d70", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["pydicom/tests/test_fileset.py::TestFileSet_Modify", "pydicom/tests/test_fileset.py::TestFileSet_Copy"], "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "model_alias": "sonnet-5", "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pydicom__pydicom-1428__sonnet-5__a1__20261021T122030787782Z", "task_id": "pydicom__pydicom-1428", "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "feature_implementation", "cost_incomplete": false, "cost_usd": 0.48091325, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 24.186968492998858, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "finished_at": "2026-10-21T12:24:46.699415+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.0044597610031, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 21, "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1428__opus-5__a1__20261021T122152490969Z", "solve_latency_s": 150.0044597610031, "started_at": "2026-10-21T12:21:52.490953+00:00", "success": true, "task_id": "pydicom__pydicom-1428", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 21365, "cache_read": 244444, "completion": 8998, "prompt": 265851, "reasoning": 2743}, "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1428-envfix1", "applies_on_top_of": "pydicom repo overlay (LANG/LC_ALL=C.UTF-8)", "attempt": 1, "cause": "The gold preflight ran pytest 8.3.5, which ignores nose-style setup/teardown. At base commit 674da68 TestFileSet_Modify and TestFileSet_Copy patch FileSet.__len__ to report 2176782336+ instances and restore it only in teardown(self), so the patch leaks and later file-set writes raise NotImplementedError", "corrections_file_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "cost_usd": 0.0, "dockerfile_sha256": "07708bb0743612bb86f89b92d3baf58f716383c09f60902ef9aef9eb22267d70", "env_correction_sha256": "1e40a104a84776aaef6f86c635543cc32e116333e671a54d256ac5b0655fdbcb", "evidence": ["pydicom/tests/test_fileset.py::TestFileSet_Modify", "pydicom/tests/test_fileset.py::TestFileSet_Copy"], "final_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1428:latest", "model_alias": "opus-5", "overlay_image_id": "sha256:e0a170d8e6ddbff5dbb2edda47d7cc6c43912444a00e240efa38f9c7aa76b9fa", "record_type": "env_correction_attempt_link", "repair_script_sha256": "383df72c71c7117ae9a0b5c7de06b1b798b433587995ddf4f8fdd5188aca7728", "run_id": "pydicom__pydicom-1428__opus-5__a1__20261021T122152490969Z", "task_id": "pydicom__pydicom-1428", "upstream_digest": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e", "upstream_image_id": "sha256:d574741060ac7c9f27621e325353ef203610a8cb3f7d7fe290ffb5fa43e9612e"}