diff --git a/tests/integration/cost_calculation/cases/decisions.py b/tests/integration/cost_calculation/cases/decisions.py new file mode 100644 index 00000000000..ea885e7ffcf --- /dev/null +++ b/tests/integration/cost_calculation/cases/decisions.py @@ -0,0 +1,461 @@ +"""Decisions cost cases: every Decisions provider called through /v1/decisions and /v1/systemone. + +Request and response bodies are verbatim captures of what litellm sent to and got back from the +real provider on 2026-10-10 (recorded through a forwarding proxy in front of each API), so each +case replays the provider's own usage shape against the scripted upstream. Spend is the recorded +usage at the suite cost map's rates; OpenRouter has no per-token rate and bills the `usage.cost` +it reports. +""" + +from __future__ import annotations + +from typing import Final + +from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase, ExactExpected, JsonResponse +from integration.cost_calculation.stream_parity.case import StreamParityTestCase + + +# gpt-6-luna +OPENAI_GPT_6_LUNA_DECISIONS: Final = CostTrackingTestCase( + name="gpt-6-luna-decisions", + covers="quota_management.spend_tracking.decisions.logs_cost", + model="gpt-6-luna", + endpoint="/v1/decisions", + request={ + "model": "$MODEL", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + upstream_body={ + "model": "gpt-6-luna", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "gpt-6-luna", + "answers": [{"type": "predicate", "name": "is_refund", "probability": 1.0}], + "usage": { + "input_tokens": 167, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 167, + }, + }, + ), + expected=ExactExpected( + spend=1.67e-05, + input_cost=1.67e-05, + output_cost=0.0, + prompt_tokens=167, + completion_tokens=0, + ), +) +OPENAI_GPT_6_LUNA_SYSTEMONE: Final = CostTrackingTestCase( + name="gpt-6-luna-systemone", + covers="quota_management.spend_tracking.systemone.logs_cost", + model="gpt-6-luna", + endpoint="/v1/systemone", + request={ + "model": "$MODEL", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + upstream_body={ + "model": "gpt-6-luna", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "gpt-6-luna", + "answers": [{"type": "predicate", "name": "is_refund", "probability": 1.0}], + "usage": { + "input_tokens": 167, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 167, + }, + }, + ), + expected=ExactExpected( + spend=1.67e-05, + input_cost=1.67e-05, + output_cost=0.0, + prompt_tokens=167, + completion_tokens=0, + ), +) + +# typesafe/jev-latest +TYPESAFE_JEV_LATEST_DECISIONS: Final = CostTrackingTestCase( + name="typesafe/jev-latest-decisions", + covers="quota_management.spend_tracking.decisions.logs_cost", + model="typesafe/jev-latest", + endpoint="/v1/decisions", + request={ + "model": "$MODEL", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + upstream_body={ + "model": "jev-latest", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "jev-1.13.0", + "answers": {"is_refund": {"type": "noul", "noul": 0.99}}, + "usage": {"input_tokens": 285, "output_tokens": 22}, + }, + ), + expected=ExactExpected( + spend=1.197e-05, + input_cost=1.197e-05, + output_cost=0.0, + prompt_tokens=285, + completion_tokens=22, + ), +) +TYPESAFE_JEV_LATEST_SYSTEMONE: Final = CostTrackingTestCase( + name="typesafe/jev-latest-systemone", + covers="quota_management.spend_tracking.systemone.logs_cost", + model="typesafe/jev-latest", + endpoint="/v1/systemone", + request={ + "model": "$MODEL", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + upstream_body={ + "model": "jev-latest", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "jev-1.13.0", + "answers": {"is_refund": {"type": "noul", "noul": 0.99}}, + "usage": {"input_tokens": 285, "output_tokens": 22}, + }, + ), + expected=ExactExpected( + spend=1.197e-05, + input_cost=1.197e-05, + output_cost=0.0, + prompt_tokens=285, + completion_tokens=22, + ), +) + +# perplexity/pplx-decider-v1-27b +PERPLEXITY_PPLX_DECIDER_V1_27B_DECISIONS: Final = CostTrackingTestCase( + name="perplexity/pplx-decider-v1-27b-decisions", + covers="quota_management.spend_tracking.decisions.logs_cost", + model="perplexity/pplx-decider-v1-27b", + endpoint="/v1/decisions", + request={ + "model": "$MODEL", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + upstream_body={ + "model": "pplx-decider-v1-27b", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "pplx-decider-v1-27b", + "answers": {"is_refund": {"type": "noul", "noul": 0.9997319426472651}}, + "usage": {"input_tokens": 98, "output_tokens": 1}, + }, + ), + expected=ExactExpected( + spend=3.92e-06, + input_cost=3.92e-06, + output_cost=0.0, + prompt_tokens=98, + completion_tokens=1, + ), +) +PERPLEXITY_PPLX_DECIDER_V1_27B_SYSTEMONE: Final = CostTrackingTestCase( + name="perplexity/pplx-decider-v1-27b-systemone", + covers="quota_management.spend_tracking.systemone.logs_cost", + model="perplexity/pplx-decider-v1-27b", + endpoint="/v1/systemone", + request={ + "model": "$MODEL", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + upstream_body={ + "model": "pplx-decider-v1-27b", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "pplx-decider-v1-27b", + "answers": {"is_refund": {"type": "noul", "noul": 0.9997319426472651}}, + "usage": {"input_tokens": 98, "output_tokens": 1}, + }, + ), + expected=ExactExpected( + spend=3.92e-06, + input_cost=3.92e-06, + output_cost=0.0, + prompt_tokens=98, + completion_tokens=1, + ), +) + +# cloudflare/@cf/cloudflare/clef-flash +CLOUDFLARE_CLEF_FLASH_DECISIONS: Final = CostTrackingTestCase( + name="cloudflare/@cf/cloudflare/clef-flash-decisions", + covers="quota_management.spend_tracking.decisions.logs_cost", + model="cloudflare/@cf/cloudflare/clef-flash", + endpoint="/v1/decisions", + request={ + "model": "$MODEL", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + upstream_body={ + "model": "clef-flash", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "result": { + "model": "clef-flash", + "answers": {"is_refund": {"type": "noul", "noul": 0.9693}}, + "usage": {"input_tokens": 157, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + ), + expected=ExactExpected( + spend=1.413e-05, + input_cost=1.413e-05, + output_cost=0.0, + prompt_tokens=157, + completion_tokens=0, + ), +) +CLOUDFLARE_CLEF_FLASH_SYSTEMONE: Final = CostTrackingTestCase( + name="cloudflare/@cf/cloudflare/clef-flash-systemone", + covers="quota_management.spend_tracking.systemone.logs_cost", + model="cloudflare/@cf/cloudflare/clef-flash", + endpoint="/v1/systemone", + request={ + "model": "$MODEL", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + upstream_body={ + "model": "clef-flash", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "result": { + "model": "clef-flash", + "answers": {"is_refund": {"type": "noul", "noul": 0.9693}}, + "usage": {"input_tokens": 157, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + ), + expected=ExactExpected( + spend=1.413e-05, + input_cost=1.413e-05, + output_cost=0.0, + prompt_tokens=157, + completion_tokens=0, + ), +) + +# databricks/databricks-openjev-qwen35-4b +DATABRICKS_OPENJEV_QWEN35_4B_DECISIONS: Final = CostTrackingTestCase( + name="databricks/databricks-openjev-qwen35-4b-decisions", + covers="quota_management.spend_tracking.decisions.logs_cost", + model="databricks/databricks-openjev-qwen35-4b", + endpoint="/v1/decisions", + request={ + "model": "$MODEL", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + upstream_body={ + "model": "databricks-openjev-qwen35-4b", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "databricks-openjev-qwen35-4b", + "answers": {"is_refund": {"type": "noul", "noul": 0.9046505248181417}}, + "usage": {"input_tokens": 45, "output_tokens": 0}, + }, + ), + expected=ExactExpected( + spend=0.0, + input_cost=0.0, + output_cost=0.0, + prompt_tokens=45, + completion_tokens=0, + ), +) +DATABRICKS_OPENJEV_QWEN35_4B_SYSTEMONE: Final = CostTrackingTestCase( + name="databricks/databricks-openjev-qwen35-4b-systemone", + covers="quota_management.spend_tracking.systemone.logs_cost", + model="databricks/databricks-openjev-qwen35-4b", + endpoint="/v1/systemone", + request={ + "model": "$MODEL", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + upstream_body={ + "model": "databricks-openjev-qwen35-4b", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "databricks-openjev-qwen35-4b", + "answers": {"is_refund": {"type": "noul", "noul": 0.9046505248181417}}, + "usage": {"input_tokens": 45, "output_tokens": 0}, + }, + ), + expected=ExactExpected( + spend=0.0, + input_cost=0.0, + output_cost=0.0, + prompt_tokens=45, + completion_tokens=0, + ), +) + +# openrouter/typesafe/jev-1.13 +OPENROUTER_TYPESAFE_JEV_1_13_DECISIONS: Final = CostTrackingTestCase( + name="openrouter/typesafe/jev-1.13-decisions", + covers="quota_management.spend_tracking.decisions.logs_cost", + model="openrouter/typesafe/jev-1.13", + endpoint="/v1/decisions", + request={ + "model": "$MODEL", + "input": "My order arrived broken and I want a refund today.", + "questions": [ + {"type": "predicate", "name": "is_refund", "instructions": "Is the customer asking for a refund?"} + ], + }, + upstream_body={ + "model": "typesafe/jev-1.13", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "typesafe/jev-1.13-20260917", + "answers": {"is_refund": {"type": "noul", "noul": 0.99}}, + "usage": {"input_tokens": 285, "output_tokens": 22, "cost": 1.197e-05}, + "id": "gen-dec-$REQUEST_ID", + "provider": "TypeSafe", + }, + ), + expected=ExactExpected( + spend=1.197e-05, + input_cost=1.197e-05, + output_cost=0.0, + prompt_tokens=285, + completion_tokens=22, + breakdown_persisted=False, + ), +) +OPENROUTER_TYPESAFE_JEV_1_13_SYSTEMONE: Final = CostTrackingTestCase( + name="openrouter/typesafe/jev-1.13-systemone", + covers="quota_management.spend_tracking.systemone.logs_cost", + model="openrouter/typesafe/jev-1.13", + endpoint="/v1/systemone", + request={ + "model": "$MODEL", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + upstream_body={ + "model": "typesafe/jev-1.13", + "state": "My order arrived broken and I want a refund today.", + "questions": {"is_refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"}}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "typesafe/jev-1.13-20260917", + "answers": {"is_refund": {"type": "noul", "noul": 0.99}}, + "usage": {"input_tokens": 285, "output_tokens": 22, "cost": 1.197e-05}, + "id": "gen-dec-$REQUEST_ID", + "provider": "TypeSafe", + }, + ), + expected=ExactExpected( + spend=1.197e-05, + input_cost=1.197e-05, + output_cost=0.0, + prompt_tokens=285, + completion_tokens=22, + breakdown_persisted=False, + ), +) + +CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + OPENAI_GPT_6_LUNA_DECISIONS, + OPENAI_GPT_6_LUNA_SYSTEMONE, + TYPESAFE_JEV_LATEST_DECISIONS, + TYPESAFE_JEV_LATEST_SYSTEMONE, + PERPLEXITY_PPLX_DECIDER_V1_27B_DECISIONS, + PERPLEXITY_PPLX_DECIDER_V1_27B_SYSTEMONE, + CLOUDFLARE_CLEF_FLASH_DECISIONS, + CLOUDFLARE_CLEF_FLASH_SYSTEMONE, + DATABRICKS_OPENJEV_QWEN35_4B_DECISIONS, + DATABRICKS_OPENJEV_QWEN35_4B_SYSTEMONE, + OPENROUTER_TYPESAFE_JEV_1_13_DECISIONS, + OPENROUTER_TYPESAFE_JEV_1_13_SYSTEMONE, +) + +PARITY: Final[tuple[StreamParityTestCase, ...]] = () diff --git a/tests/integration/cost_calculation/cases/misc.py b/tests/integration/cost_calculation/cases/misc.py index d1c17b11adb..61dae9a3b53 100644 --- a/tests/integration/cost_calculation/cases/misc.py +++ b/tests/integration/cost_calculation/cases/misc.py @@ -1,4 +1,4 @@ -"""azure_ai, bedrock, cohere, cohere_chat, dashscope, deepgram, deepseek, groq, mistral, openrouter, perplexity, text-completion-openai, xai cost tracking cases, one CostTrackingTestCase literal per request shape (moved from cost_tracking_cases.json). +"""azure_ai, bedrock, cohere, cohere_chat, dashscope, deepgram, deepseek, groq, mistral, openrouter, text-completion-openai, xai cost tracking cases, one CostTrackingTestCase literal per request shape (moved from cost_tracking_cases.json). PARITY pairs the streamed case with its plain twin where both bill the same row.""" @@ -840,31 +840,6 @@ MISTRAL_MEDIUM_2604_JSON: Final = CostTrackingTestCase( ) -# perplexity/pplx-decider-v1-27b -PERPLEXITY_PPLX_DECIDER_V1_27B_DECISIONS: Final = CostTrackingTestCase( - name="perplexity/pplx-decider-v1-27b-decisions", - covers="quota_management.spend_tracking.decisions_costs", - model="perplexity/pplx-decider-v1-27b", - endpoint="/v1/systemone", - request={ - "model": "$MODEL", - "state": {"source": "cost-tracking"}, - "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, - }, - response=JsonResponse( - content_type="application/json", - body={ - "model": "pplx-decider-v1-27b", - "answers": {"is_defect": {"type": "noul", "noul": 0.9}}, - "usage": {"input_tokens": 367, "output_tokens": 3}, - }, - ), - expected=ExactExpected( - spend=1.468e-05, input_cost=1.468e-05, output_cost=0.0, prompt_tokens=367, completion_tokens=3 - ), -) - - # xai/grok-4.7 XAI_GROK_4_7_INPUT_TEXT: Final = CostTrackingTestCase( name="xai-grok-4.7-input_text", @@ -925,7 +900,6 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GROQ_QWEN_3_8_STREAM_X_GROQ_RECOUNT, COHERE_COMMAND_A_V2_TOKENS, MISTRAL_MEDIUM_2604_JSON, - PERPLEXITY_PPLX_DECIDER_V1_27B_DECISIONS, XAI_GROK_4_7_INPUT_TEXT, ) diff --git a/tests/integration/cost_calculation/catalog.py b/tests/integration/cost_calculation/catalog.py index fe7367aed8a..2710de5e5ea 100644 --- a/tests/integration/cost_calculation/catalog.py +++ b/tests/integration/cost_calculation/catalog.py @@ -10,6 +10,7 @@ from integration.cost_calculation.cases import ( azure, batch_realtime, bedrock_converse, + decisions, fireworks_ai, gemini, misc, @@ -36,6 +37,7 @@ _PROVIDER_MODULES: Final = ( anthropic, gemini, bedrock_converse, + decisions, vertex_ai, azure, fireworks_ai, diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 2b597aaadb8..6c10df5f423 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -43,6 +43,37 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "search_context_size_high": 0.012, }, }, + "gpt-6-luna": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1e-08, + }, + "typesafe/jev-latest": { + "litellm_provider": "typesafe", + "mode": "evaluation", + "input_cost_per_token": 4.2e-08, + "output_cost_per_token": 0.0, + }, + "cloudflare/@cf/cloudflare/clef-flash": { + "litellm_provider": "cloudflare", + "mode": "evaluation", + "input_cost_per_token": 9e-08, + "output_cost_per_token": 0.0, + }, + "databricks/databricks-openjev-qwen35-4b": { + "litellm_provider": "databricks", + "mode": "evaluation", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + "openrouter/typesafe/jev-1.13": { + "litellm_provider": "openrouter", + "mode": "evaluation", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, "perplexity/pplx-decider-v1-27b": { "input_cost_per_token": 4e-08, "litellm_provider": "perplexity", diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index a146401f3ae..d672edb03f0 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -250,6 +250,7 @@ class CostTrackingTestCase(BaseModel): "/v1/images/generations", "/v1/images/edits", "/v1/systemone", + "/v1/decisions", ] | Annotated[str, Field(pattern=r"^/(gemini|anthropic|bedrock)/")] ) = "/v1/chat/completions" @@ -260,6 +261,7 @@ class CostTrackingTestCase(BaseModel): expected: Expected fallback_from: StoredResponse | None = None disconnect_after_frames: int | None = Field(default=None, ge=1) + upstream_body: dict[str, JsonValue] | None = None @property def rates(self) -> CostMapEntry: @@ -504,6 +506,9 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( "dashscope": "", "openrouter": "", "perplexity": "", + "typesafe": "typesafe", + "cloudflare": "cloudflare", + "databricks": "databricks", "deepseek": "", "xai": "", "azure_ai": "azure_ai", @@ -548,6 +553,9 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( "openai": MappingProxyType({}), "dashscope": MappingProxyType({}), "openrouter": MappingProxyType({}), + "typesafe": MappingProxyType({}), + "cloudflare": MappingProxyType({}), + "databricks": MappingProxyType({}), "perplexity": MappingProxyType({}), "deepseek": MappingProxyType({}), "xai": MappingProxyType({}), diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index a84c31e810d..9db55589fb3 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -252,32 +252,19 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert row.spend == 0, f"{case.name}: failure spend was {row.spend}" return assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" - if case.endpoint == "/v1/systemone": + if case.upstream_body is not None: observed: Final = JSON_OBJECT.validate_json( httpx.get(f"{gateway.upstream_url}/__observations", timeout=5, trust_env=False).content ) - decision_observations: Final = tuple( - value + upstream_bodies: Final = tuple( + value.get("body") for value in observed["requests"] - if isinstance(value, dict) and value.get("path") == f"/{scenario_id}/v1/decisions" + if isinstance(value, dict) + and value.get("method") == "POST" + and str(value.get("path", "")).startswith(f"/{scenario_id}/") ) - assert decision_observations == ( - { - "path": f"/{scenario_id}/v1/decisions", - "authorization": "Bearer sk-scripted-provider", - "method": "POST", - "api_key": "", - "body": { - "model": "pplx-decider-v1-27b", - "state": {"source": "cost-tracking"}, - "questions": { - "is_defect": { - "type": "noul", - "instructions": "Is this a defect?", - } - }, - }, - }, + assert upstream_bodies == (case.upstream_body,), ( + f"{case.name}: upstream received {upstream_bodies}, expected exactly {case.upstream_body}" ) if case.response.content_type == "text/event-stream": _assert_stream_has_no_error(response.text)