From 7143366437d05b896ef5cf774734428ef2fe0a38 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:47:32 -0700 Subject: [PATCH] test(decisions): bill the OpenAI-format openrouter case from provider-reported cost (#45853) PR #45667 removed openrouter/typesafe/jev-1.13 from the cost map after #45640 moved OpenRouter decisions billing to usage.cost. The OpenAI-format wire test still read the removed row and raised KeyError on CircleCI integration-providers Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 5894968c5dc31dad5a20ff27750b27f399b0576f) --- .../test_decisions_openai_format_wire.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index f0e7163e264..1104a6151db 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -105,19 +105,25 @@ class _Provider: wraps_result: bool cost_map_key: str | None speaks_openai: bool = False + provider_reported_cost: float | None = None def upstream_body(self) -> dict[str, JsonValue]: if self.speaks_openai: return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS} return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + def _with_reported_cost(self, usage: dict[str, JsonValue]) -> dict[str, JsonValue]: + if self.provider_reported_cost is None: + return usage + return {**usage, "cost": self.provider_reported_cost} + def upstream_reply(self) -> dict[str, JsonValue]: if self.speaks_openai: - return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + return {"model": self.body_model, "answers": _ANSWERS, "usage": self._with_reported_cost(_USAGE)} answer: Final[dict[str, JsonValue]] = { "model": self.body_model, "answers": _SYSTEM_ONE_ANSWERS, - "usage": _SYSTEM_ONE_USAGE, + "usage": self._with_reported_cost(_SYSTEM_ONE_USAGE), } return {"result": answer, "success": True} if self.wraps_result else answer @@ -143,7 +149,8 @@ _PROVIDERS: Final = ( "typesafe/jev-1.13", _API_KEY, False, - "openrouter/typesafe/jev-1.13", + None, + provider_reported_cost=1.5834e-5, ), _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None @@ -235,11 +242,13 @@ def _number(value: JsonValue) -> float: return float(value) -def _expected_spend(cost_map_key: str | None) -> float: - if cost_map_key is None: +def _expected_spend(provider: _Provider) -> float: + if provider.provider_reported_cost is not None: + return provider.provider_reported_cost + if provider.cost_map_key is None: return 0.0 cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) - prices: Final = object_value(cost_map[cost_map_key]) + prices: Final = object_value(cost_map[provider.cost_map_key]) return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( prices["output_cost_per_token"] ) @@ -320,10 +329,8 @@ def _drop_params_config(directory: Path, api_base: str) -> Path: @pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id) -def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( - gateway: Gateway, provider: _Provider -) -> None: - expected_spend: Final = _expected_spend(provider.cost_map_key) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None: + expected_spend: Final = _expected_spend(provider) with gateway.scenario() as scenario: handle: Final = _register(scenario, provider.upstream_reply()) model: Final = _deployment(scenario, handle, provider)