scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 199

chore(cookbook): profiling checkpoint after batch 5

Browse source 
Co-Authored-By: bot_apk <apk@cognition.ai>
This commit is contained in:
Devin AI 2026-10-21 05:31:26 +00:00
parent 2335b16b44
commit 666c431d64
2 changed files with 465 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

420
cookbook/capability_routing_pilot/profiling/checkpoints/batch_005/capability_map_intermediate.json Normal file
View file

@ -0,0 +1,420 @@
{
"category_coverage": {
"algorithms_data": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"feature_implementation": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"repository_debugging": {
"profile_count": 3,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 3,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"shell_build": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
},
"unknown": {
"profile_count": 0,
"status_counts": {
"evidenced_below_target": 0,
"insufficient_evidence": 0,
"no_evidence": 0,
"qualified": 0,
"unsupported_zero_success": 0
}
}
},
"category_decisions": {
"algorithms_data": {
"decision": "unsupported_category",
"reason": "unsupported_category",
"selected_profile": null,
"threshold_unmet": false
},
"feature_implementation": {
"decision": "no_qualified_model",
"reason": "no_qualified_model",
"selected_profile": null,
"threshold_unmet": false
},
"repository_debugging": {
"decision": "no_qualified_model",
"reason": "no_qualified_model",
"selected_profile": null,
"threshold_unmet": false
},
"shell_build": {
"decision": "unsupported_category",
"reason": "unsupported_category",
"selected_profile": null,
"threshold_unmet": false
},
"unknown": {
"decision": "unsupported_category",
"reason": "unsupported_category",
"selected_profile": null,
"threshold_unmet": false
}
},
"final": false,
"infra_deficits": [],
"interpretation": "Beta(1,1) credible bound describes the exchangeable public-dev task distribution for this category, not guaranteed per-task success",
"min_evidence_note": "necessary not sufficient: 13/13 is the minimum perfect cell clearing 0.80; not a general adequacy claim",
"preflight_failed": [
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest",
"latency_s": 36.985300702999666,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1413",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460",
"upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest",
"latency_s": 45.86221657400165,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1439",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517",
"upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"
},
{
"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest",
"latency_s": 59.88555217099929,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"haiku-4.5",
"sonnet-5",
"opus-5"
],
"noop_status": "unresolved",
"overlay_image_id": null,
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pyvista__pyvista-4329",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3",
"upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest",
"latency_s": 37.34210575900215,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-1720",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337",
"upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"
},
{
"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-995",
"category": "repository_debugging",
"cost_usd": 0.0,
"error_class": "env",
"final_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444",
"gold_status": "unresolved",
"grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-995:latest",
"latency_s": 37.53449416300282,
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_order": [
"sonnet-5",
"opus-5",
"haiku-4.5"
],
"noop_status": "unresolved",
"overlay_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444",
"record_type": "preflight_failed",
"status": "preflight_failed",
"task_id": "pydicom__pydicom-995",
"tenant_id": "public_dev_poc",
"upstream_digest": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39",
"upstream_image_id": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39"
}
],
"preflight_incomplete": [
"pydicom__pydicom-1334",
"pylint-dev__astroid-1351",
"pvlib__pvlib-python-980",
"pydicom__pydicom-1694",
"pvlib__pvlib-python-1089",
"sqlfluff__sqlfluff-1763",
"pylint-dev__astroid-2309",
"pydicom__pydicom-1256",
"pvlib__pvlib-python-1666",
"pylint-dev__astroid-1978",
"pydicom__pydicom-1365",
"pydicom__pydicom-933",
"pyvista__pyvista-4414",
"pydicom__pydicom-1048",
"pylint-dev__astroid-1333",
"pyvista__pyvista-3675",
"pvlib__pvlib-python-1469",
"sqlfluff__sqlfluff-905",
"pvlib__pvlib-python-1186",
"pvlib__pvlib-python-1764",
"pvlib__pvlib-python-1073",
"pvlib__pvlib-python-1852",
"marshmallow-code__marshmallow-1702",
"pvlib__pvlib-python-907",
"pyvista__pyvista-432",
"pydicom__pydicom-1017",
"pydicom__pydicom-1241",
"pydicom__pydicom-1428",
"pydicom__pydicom-1228",
"pvlib__pvlib-python-718",
"pvlib__pvlib-python-1176",
"pvlib__pvlib-python-1181",
"pylint-dev__astroid-1616",
"pvlib__pvlib-python-823",
"pvlib__pvlib-python-1478",
"pylint-dev__astroid-1417",
"pvlib__pvlib-python-1682",
"pyvista__pyvista-4648",
"pyvista__pyvista-4311",
"pvlib__pvlib-python-1426",
"marshmallow-code__marshmallow-2123",
"pvlib__pvlib-python-1458",
"pvlib__pvlib-python-807",
"pvlib__pvlib-python-1395",
"pvlib__pvlib-python-1832"
],
"profiles": [
{
"attempt_count": 10,
"beta_mean": 0.5,
"cost_per_graded_task_usd": 0.179941745,
"graded_count": 10,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "repository_debugging",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "haiku-4.5",
"returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.27124991407766885,
"mean_latency_s": 146.00481095260002,
"min_evidence": 13,
"primary_labels": [
"repository_debugging"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "haiku-4.5"
},
"raw_rate": 0.5,
"secondary_labels": [],
"status": "insufficient_evidence",
"successes": 5,
"tau": 0.8,
"total_spend_usd": 1.79941745
},
{
"attempt_count": 10,
"beta_mean": 0.5833333333333334,
"cost_per_graded_task_usd": 0.42326115,
"graded_count": 10,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "repository_debugging",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "opus-5",
"returned_model_version": "litellm_proxy/anthropic/claude-opus-5",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.3498115345719175,
"mean_latency_s": 137.00437219750057,
"min_evidence": 13,
"primary_labels": [
"repository_debugging"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "opus-5"
},
"raw_rate": 0.6,
"secondary_labels": [],
"status": "insufficient_evidence",
"successes": 6,
"tau": 0.8,
"total_spend_usd": 4.2326115
},
{
"attempt_count": 10,
"beta_mean": 0.6666666666666666,
"cost_per_graded_task_usd": 0.19906978000000003,
"graded_count": 10,
"infra_count": 0,
"key": {
"budget_version": "budget_1usd_50_600_v1",
"category": "repository_debugging",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_alias": "sonnet-5",
"returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5",
"tenant_id": "public_dev_poc"
},
"lower_5pct": 0.43562581171077036,
"mean_latency_s": 127.00538523540017,
"min_evidence": 13,
"primary_labels": [
"repository_debugging"
],
"provenance": {
"alias_note": "Model aliases can resolve to changing deployments; returned model versions and deployment IDs are isolated",
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d",
"manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4",
"model_alias": "sonnet-5"
},
"raw_rate": 0.7,
"secondary_labels": [],
"status": "insufficient_evidence",
"successes": 7,
"tau": 0.8,
"total_spend_usd": 1.9906978000000002
}
],
"quarantined_attempts": [],
"tenant_id": "public_dev_poc",
"thresholds": {
"min_evidence": 13,
"tau": 0.8
},
"versions": {
"budget_version": "budget_1usd_50_600_v1",
"harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43",
"harness_version": "run_pilot_attempt_v1",
"model_deployments": {
"haiku-4.5": [
"13ef3d99-7edf-4a6b-aab6-f088c8e25964"
],
"opus-5": [
"13ef3d99-7edf-4a6b-aab6-f088c8e25964"
],
"sonnet-5": [
"13ef3d99-7edf-4a6b-aab6-f088c8e25964"
]
},
"model_versions": {
"haiku-4.5": [
"litellm_proxy/anthropic/claude-haiku-4-5-20251001"
],
"opus-5": [
"litellm_proxy/anthropic/claude-opus-5"
],
"sonnet-5": [
"litellm_proxy/anthropic/claude-sonnet-5"
]
},
"observed_model_cohorts": [
{
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"model_alias": "haiku-4.5",
"returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001"
},
{
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"model_alias": "opus-5",
"returned_model_version": "litellm_proxy/anthropic/claude-opus-5"
},
{
"deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964",
"model_alias": "sonnet-5",
"returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5"
}
],
"version_changes": []
}
}

45
cookbook/capability_routing_pilot/profiling/checkpoints/batch_005/task_runs.jsonl Normal file
View file

@ -0,0 +1,45 @@
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1413", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1413:latest", "latency_s": 36.985300702999666, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:410e2f40cb23aa1923369affad114b15e9ef2eb5f079997cfa8ebd9dd7d6874a", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1413", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460", "upstream_image_id": "sha256:1e091a2e42b92bec0be650972b22a320489225c4258e5211c07c9f592a9a2460"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "latency_s": 37.443159876998834, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "latency_s": 42.63817018199916, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15246375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.22013637100099, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:29:42.514070+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.00210723499913, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 11, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1216__opus-5__a1__20261021T042841152530Z", "solve_latency_s": 40.00210723499913, "started_at": "2026-10-21T04:28:41.152510+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11155, "cache_read": 72220, "completion": 1861, "prompt": 83397, "reasoning": 34}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0754433, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.261532576998434, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:29:49.569191+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.00336846000209, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 15, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-1192__sonnet-5__a1__20261021T042841151566Z", "solve_latency_s": 50.00336846000209, "started_at": "2026-10-21T04:28:41.151547+00:00", "success": true, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11487, "cache_read": 101329, "completion": 2640, "prompt": 112846, "reasoning": 66}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.08664235, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.42284784900039, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:31:34.083967+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00328306499796, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 25, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1216__haiku-4.5__a1__20261021T042942515886Z", "solve_latency_s": 90.00328306499796, "started_at": "2026-10-21T04:29:42.515874+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17291, "cache_read": 192516, "completion": 6628, "prompt": 222444, "reasoning": 0}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.38297225, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 20.581349566000426, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:32:00.355179+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 110.00375107299988, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 20, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-1192__opus-5__a1__20261021T042949571022Z", "solve_latency_s": 110.00375107299988, "started_at": "2026-10-21T04:29:49.571011+00:00", "success": true, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 17105, "cache_read": 226732, "completion": 6500, "prompt": 243877, "reasoning": 3366}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1216", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0498986, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.37230331899991, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "finished_at": "2026-10-21T04:32:35.584861+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1216:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 40.00220326099952, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 10, "overlay_image_id": "sha256:bb043aabbce3519f8fa9335eeaba9ffe3174baa23892c807aeec80aae9228512", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1216__sonnet-5__a1__20261021T043134086373Z", "solve_latency_s": 40.00220326099952, "started_at": "2026-10-21T04:31:34.086359+00:00", "success": true, "task_id": "pvlib__pvlib-python-1216", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8744, "cache_read": 53193, "completion": 1736, "prompt": 61957, "reasoning": 10}, "upstream_digest": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a", "upstream_image_id": "sha256:78f338c8ad90f06b348db68087156e0a6d729566f07c9d7bc2e8a60d07b66f9a"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1192", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2071005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "finished_at": "2026-10-21T04:34:50.522036+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1192:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 170.0046138639991, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:e0a4b7b0fed20d41627aa38f48fe9606c494ee160ff2e427b4e8cc31a07fcebb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-1192__haiku-4.5__a1__20261021T043200357212Z", "solve_latency_s": 170.0046138639991, "started_at": "2026-10-21T04:32:00.357202+00:00", "success": false, "task_id": "pydicom__pydicom-1192", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 31386, "cache_read": 742450, "completion": 15459, "prompt": 790164, "reasoning": 0}, "upstream_digest": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033", "upstream_image_id": "sha256:3347b398b888363819198adfacc909464b5a8c25b7cf6e41987c76ecb25fb033"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1439", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1439:latest", "latency_s": 45.86221657400165, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:0acd7d5d280d24e1fc4c7b9cea90c665ba89fdefb25c56b74d681ba9d4b707b1", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1439", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517", "upstream_image_id": "sha256:ddb010dc7d015d6cc7c279d3efcdc756b9f4ed90eb254a004dc78e7d57db5517"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "latency_s": 46.50070145599966, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "latency_s": 39.32556367899815, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.6126465, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:47:21.107529+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 200.00617045400213, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 27, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-2307__opus-5__a1__20261021T044400932430Z", "solve_latency_s": 200.00617045400213, "started_at": "2026-10-21T04:44:00.932414+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23326, "cache_read": 365428, "completion": 11355, "prompt": 388808, "reasoning": 6719}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.3954644, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 22.276458722000825, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:48:33.342443+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.0124097070002, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 32, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1165__sonnet-5__a1__20261021T044400931150Z", "solve_latency_s": 250.0124097070002, "started_at": "2026-10-21T04:44:00.931128+00:00", "success": true, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 30230, "cache_read": 513457, "completion": 21707, "prompt": 543751, "reasoning": 15209}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1836434, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:50:01.255188+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 160.00828153600014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-2307__haiku-4.5__a1__20261021T044721112372Z", "solve_latency_s": 160.00828153600014, "started_at": "2026-10-21T04:47:21.112358+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 29088, "cache_read": 697624, "completion": 12455, "prompt": 741958, "reasoning": 0}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.626164, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:51:33.536769+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00563653500285, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 13, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1165__opus-5__a1__20261021T044833345077Z", "solve_latency_s": 180.00563653500285, "started_at": "2026-10-21T04:48:33.345065+00:00", "success": false, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 24668, "cache_read": 184418, "completion": 15186, "prompt": 209112, "reasoning": 12703}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1165", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.26214350000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "finished_at": "2026-10-21T04:55:23.672687+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1165:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 230.00504096200166, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 50, "overlay_image_id": "sha256:3d52ef3190c3ce261638416008b5f5cf865c306b415157ac8821ac72f5f833a8", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1165__haiku-4.5__a1__20261021T045133539818Z", "solve_latency_s": 230.00504096200166, "started_at": "2026-10-21T04:51:33.539806+00:00", "success": false, "task_id": "pvlib__pvlib-python-1165", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 43394, "cache_read": 976470, "completion": 19432, "prompt": 1032958, "reasoning": 0}, "upstream_digest": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1", "upstream_image_id": "sha256:71e033964894077384c0c27a04ae0271e28eaad35602aa7570d40c683f1967d1"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.5375916000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "finished_at": "2026-10-21T04:55:41.469860+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-2307:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 340.0079844879983, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-2307__sonnet-5__a1__20261021T045001257820Z", "solve_latency_s": 340.0079844879983, "started_at": "2026-10-21T04:50:01.257807+00:00", "success": false, "task_id": "pylint-dev__astroid-2307", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37278, "cache_read": 973333, "completion": 24953, "prompt": 1010711, "reasoning": 15582}, "upstream_digest": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5", "upstream_image_id": "sha256:1b4a0dbcbd8dd161ba537c19c80c7da713baa543fece8caa87d5d32483bb88b5"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pyvista_1776_pyvista-4329:latest", "latency_s": 59.88555217099929, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pyvista__pyvista-4329", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3", "upstream_image_id": "sha256:424ecdb4949d49836f004ccccfaf59c2207d0bdd87d8151ecfa92ba3eaf91eb3"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-1720", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-1720:latest", "latency_s": 37.34210575900215, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:c931326698b89fee911ab9e586b2578cb671eb39c880799b07fa05d5ceb59297", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-1720", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337", "upstream_image_id": "sha256:389d351b51bc10da28fabc12c1afb07053ecb1c7459602f012761b7d1880b337"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "latency_s": 36.9401233719982, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.49552525000000003, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.36470413599818, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:01:43.053923+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00416897800096, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 19, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pydicom__pydicom-897__opus-5__a1__20261021T045854563009Z", "solve_latency_s": 150.00416897800096, "started_at": "2026-10-21T04:58:54.562993+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19647, "cache_read": 236183, "completion": 10178, "prompt": 255868, "reasoning": 5998}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.15681435, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:03:53.181859+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.0038786289988, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 50, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pydicom__pydicom-897__haiku-4.5__a1__20261021T050143057007Z", "solve_latency_s": 130.0038786289988, "started_at": "2026-10-21T05:01:43.056996+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 23533, "cache_read": 670341, "completion": 9320, "prompt": 707638, "reasoning": 0}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-897", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.2344321, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 18.46944500400059, "eval_status": "unresolved", "exit_status": "Submitted", "final_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "finished_at": "2026-10-21T05:06:41.825678+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-897:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00388437999936, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 45, "overlay_image_id": "sha256:3d11bd0a308d5776e20649da5399e76ad0ca58a375d9cd8cdb0ecb37d29f58cb", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pydicom__pydicom-897__sonnet-5__a1__20261021T050353184991Z", "solve_latency_s": 150.00388437999936, "started_at": "2026-10-21T05:03:53.184978+00:00", "success": false, "task_id": "pydicom__pydicom-897", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18927, "cache_read": 459623, "completion": 9501, "prompt": 478640, "reasoning": 3384}, "upstream_digest": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d", "upstream_image_id": "sha256:9d2515d9846d3190408a29d70d25654095eef197d45fc02642c21ffe9e7e1a7d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "latency_s": 39.889730383001734, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "routing_poc/profiling-overlay:pydicom__pydicom-995", "category": "repository_debugging", "cost_usd": 0.0, "error_class": "env", "final_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", "gold_status": "unresolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pydicom_1776_pydicom-995:latest", "latency_s": 37.53449416300282, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": "sha256:f9b967d3213df7a453959b4bd085cbb1b054262d7c5d39d8d142aa2adb52d444", "record_type": "preflight_failed", "status": "preflight_failed", "task_id": "pydicom__pydicom-995", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39", "upstream_image_id": "sha256:ee9f71cd4126fb421c95f6ef3d23dd15e732f140e16332065df590d069c6ec39"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "latency_s": 39.2809497240014, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.32255725, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.878957515000366, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:11:02.894305+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 80.00347018200046, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 13, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pvlib__pvlib-python-1349__opus-5__a1__20261021T050922867062Z", "solve_latency_s": 80.00347018200046, "started_at": "2026-10-21T05:09:22.867049+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18737, "cache_read": 157992, "completion": 5053, "prompt": 176755, "reasoning": 2608}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1955686, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:11:23.004924+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 120.00499666500036, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1364__haiku-4.5__a1__20261021T050922865588Z", "solve_latency_s": 120.00499666500036, "started_at": "2026-10-21T05:09:22.865569+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 37378, "cache_read": 1011631, "completion": 7784, "prompt": 1057772, "reasoning": 0}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.25508825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.928946245003317, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:14:22.849769+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 180.00484361299823, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 46, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pvlib__pvlib-python-1349__haiku-4.5__a1__20261021T051102898250Z", "solve_latency_s": 180.00484361299823, "started_at": "2026-10-21T05:11:02.898236+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 42975, "cache_read": 1255085, "completion": 13855, "prompt": 1304646, "reasoning": 0}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "routing_poc/profiling-overlay:pvlib__pvlib-python-1349", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1126159, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.824896968002577, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "finished_at": "2026-10-21T05:15:32.696323+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pvlib_1776_pvlib-python-1349:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.0022958030022, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 14, "overlay_image_id": "sha256:8ef3899f2315d676eecc864190adcb6362bf9f658fd44bc03299501abc449a20", "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pvlib__pvlib-python-1349__sonnet-5__a1__20261021T051422853521Z", "solve_latency_s": 50.0022958030022, "started_at": "2026-10-21T05:14:22.853508+00:00", "success": true, "task_id": "pvlib__pvlib-python-1349", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 18231, "cache_read": 170812, "completion": 3282, "prompt": 189071, "reasoning": 872}, "upstream_digest": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e", "upstream_image_id": "sha256:f620d2e28a1991881d9b8bbb5c8f7b2605dd7c97a5d23802c4de165c89a9b72e"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.4375135, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "LimitsExceeded", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:16:03.478380+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 280.0084194079973, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 50, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1364__sonnet-5__a1__20261021T051123009015Z", "solve_latency_s": 280.0084194079973, "started_at": "2026-10-21T05:11:23.009002+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 35875, "cache_read": 974680, "completion": 15269, "prompt": 1010655, "reasoning": 8390}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "attempt": 1, "budget_guard_refused": true, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.648212, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "solver", "eval_latency_s": 0.0, "eval_status": "skipped_empty_patch", "exit_status": "BudgetGuard", "final_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "finished_at": "2026-10-21T05:20:13.628921+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1364:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 250.0070123559999, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1364__opus-5__a1__20261021T051603482193Z", "solve_latency_s": 250.0070123559999, "started_at": "2026-10-21T05:16:03.482180+00:00", "success": false, "task_id": "pylint-dev__astroid-1364", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 19296, "cache_read": 169404, "completion": 17710, "prompt": 188732, "reasoning": 15644}, "upstream_digest": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4", "upstream_image_id": "sha256:17f73e582ebb10fc23777d108257eec5de4f0122aa70f444fd61cac7ec0e0ce4"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "latency_s": 39.77621123099743, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "latency_s": 43.17733980900084, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "category": "repository_debugging", "cost_usd": 0.0, "error_class": null, "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "gold_status": "resolved", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "latency_s": 37.75819920799768, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "noop_status": "unresolved", "overlay_image_id": null, "record_type": "preflight_passed", "status": "preflight_passed", "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.0995685, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.37739269600206, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:23:28.637543+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.001993197998672, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 6, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1164__opus-5__a1__20261021T052240132919Z", "solve_latency_s": 30.001993197998672, "started_at": "2026-10-21T05:22:40.132907+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 7776, "cache_read": 27567, "completion": 1485, "prompt": 35355, "reasoning": 253}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.08022180000000001, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.130332180997357, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:23:51.415788+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 50.007217943002615, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 16, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "pylint-dev__astroid-1719__sonnet-5__a1__20261021T052240127823Z", "solve_latency_s": 50.007217943002615, "started_at": "2026-10-21T05:22:40.127805+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 11136, "cache_read": 111339, "completion": 3005, "prompt": 122507, "reasoning": 523}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.18115045, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.322718805000477, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:25:09.592869+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 130.00579304400162, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 46, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1810__haiku-4.5__a1__20261021T052240130927Z", "solve_latency_s": 130.00579304400162, "started_at": "2026-10-21T05:22:40.130915+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 33073, "cache_read": 697242, "completion": 10692, "prompt": 746940, "reasoning": 0}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.12290365, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 18.234139322001283, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:25:27.046567+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 100.00316618500074, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 39, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "marshmallow-code__marshmallow-1164__haiku-4.5__a1__20261021T052328641925Z", "solve_latency_s": 100.00316618500074, "started_at": "2026-10-21T05:23:28.641910+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 21985, "cache_read": 420494, "completion": 7907, "prompt": 456317, "reasoning": 0}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.035914600000000005, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.89811670599738, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:25:59.649361+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.00381014999948, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 9, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1810__sonnet-5__a1__20261021T052509597736Z", "solve_latency_s": 30.00381014999948, "started_at": "2026-10-21T05:25:09.597723+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 6304, "cache_read": 35943, "completion": 1293, "prompt": 42265, "reasoning": 40}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.031602, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.185294439001154, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "finished_at": "2026-10-21T05:26:16.378423+00:00", "gateway_model": "anthropic/claude-sonnet-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1164:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 30.00225875400065, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "sonnet-5", "model_order": ["opus-5", "haiku-4.5", "sonnet-5"], "n_calls": 8, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-sonnet-5", "run_id": "marshmallow-code__marshmallow-1164__sonnet-5__a1__20261021T052527051475Z", "solve_latency_s": 30.00225875400065, "started_at": "2026-10-21T05:25:27.051461+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1164", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 5698, "cache_read": 27125, "completion": 1190, "prompt": 32839, "reasoning": 61}, "upstream_digest": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d", "upstream_image_id": "sha256:46a2ba7f6d9fa15fa752ea1332d70abc4c63f4aa040ed316380d6fe6dd1dbc0d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.23039375, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 19.577401711001585, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "finished_at": "2026-10-21T05:27:49.390345+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.marshmallow-code_1776_marshmallow-1810:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 90.00354062100087, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["haiku-4.5", "sonnet-5", "opus-5"], "n_calls": 8, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "marshmallow-code__marshmallow-1810__opus-5__a1__20261021T052559657755Z", "solve_latency_s": 90.00354062100087, "started_at": "2026-10-21T05:25:59.657740+00:00", "success": true, "task_id": "marshmallow-code__marshmallow-1810", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 8455, "cache_read": 41590, "completion": 6267, "prompt": 50061, "reasoning": 4667}, "upstream_digest": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d", "upstream_image_id": "sha256:2cc813f08bcc2cadc5a12429e102297d9d92c5cf263de53f8ccdbc68381cf44d"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.66210825, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 23.277767100000347, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:28:14.846207+00:00", "gateway_model": "anthropic/claude-opus-5", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 240.00587134300076, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "opus-5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 27, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-opus-5", "run_id": "pylint-dev__astroid-1719__opus-5__a1__20261021T052351420018Z", "solve_latency_s": 240.00587134300076, "started_at": "2026-10-21T05:23:51.420006+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 20941, "cache_read": 368864, "completion": 13861, "prompt": 389859, "reasoning": 8383}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}
{"agent_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "attempt": 1, "budget_guard_refused": false, "budget_version": "budget_1usd_50_600_v1", "category": "repository_debugging", "cost_incomplete": false, "cost_usd": 0.1483624, "deployment_id": "13ef3d99-7edf-4a6b-aab6-f088c8e25964", "error_class": "none", "eval_latency_s": 21.447099036999134, "eval_status": "resolved", "exit_status": "Submitted", "final_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "finished_at": "2026-10-21T05:31:06.320651+00:00", "gateway_model": "anthropic/claude-haiku-4-5-20251001", "grader_image_ref": "swebench/sweb.eval.x86_64.pylint-dev_1776_astroid-1719:latest", "harness_config_hash": "32499cdda5a3167b68e297e7282f3b2bac2f4635c3cc1048c4408c9a396e4e43", "harness_sha256": "b53a6e4c72699588217fef63f48c1e2cecac45c8a77922e1f7bef1a9bebd953d", "harness_version": "run_pilot_attempt_v1", "is_infra": false, "latency_s": 150.00421196300158, "manifest_sha256": "15720dee12d40b62780837ed12a403f3951e9c5bb1ed18ec0d5d5d1f4776bff4", "model_alias": "haiku-4.5", "model_order": ["sonnet-5", "opus-5", "haiku-4.5"], "n_calls": 42, "overlay_image_id": null, "record_type": "attempt", "retry_eligible": false, "returned_model_version": "litellm_proxy/anthropic/claude-haiku-4-5-20251001", "run_id": "pylint-dev__astroid-1719__haiku-4.5__a1__20261021T052814852065Z", "solve_latency_s": 150.00421196300158, "started_at": "2026-10-21T05:28:14.852050+00:00", "success": true, "task_id": "pylint-dev__astroid-1719", "tenant_id": "public_dev_poc", "tokens": {"cache_creation": 25984, "cache_read": 629064, "completion": 9321, "prompt": 661419, "reasoning": 0}, "upstream_digest": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc", "upstream_image_id": "sha256:6d3d2ae003f8c6a35fb8001ed5f323869ba5eb915ad89458cb95ebf62acac6bc"}