From 216041e24378243a104c4ab871a5fd8a73d5fcbe Mon Sep 17 00:00:00 2001
From: Devin Gupta <55953391+devin-gupta@users.noreply.github.com>
Date: Mon, 2 Nov 2026 20:47:02 -0800
Subject: [PATCH 01/25] feat(decisions): add Parallel AI web-grounded decisions
(#49188)
* feat(decisions): add Parallel AI web-grounded decisions
* fix(decisions): name the Parallel model decision
* fix(parallel_ai): restrict environment credentials to trusted decision endpoints
* fix(decisions): keep Parallel evidence fields off other providers and handle missing confidence
Only fields a provider names reach the OpenAI response, so existing providers keep
their base shape. A malformed api_base answers 400 on both credential paths. The
complexity router classifier and the Playground handle a provider that omits
confidence. Regenerates the lazy OpenAPI snapshot and the dashboard API types
* chore(cost-map): declare the Decisions routes on parallel_ai/decision
The provider endpoint files are generated from the cost map, so the row must
name both routes for the generated System One flag to hold
* fix(decisions): report no classifier confidence when a provider omits it
A choice answer with no confidence is a valid classifier answer, so the
complexity router picks its tier and leaves classifier_confidence and the
confidence signal out instead of reporting a different metric in its place.
Also show the Parallel AI logo in the provider picker and add a Parallel
translation fixture captured from the live API for both Decisions routes.
* fix(decisions): keep a token-priced row on token billing when it also sets a query price
A plain input_cost_per_query bills flat only on a row with no token price, and input_cost_per_query_decisions always bills flat, so a custom Decisions row that already bills tokens costs the same as before this PR
* fix(decisions): bill a deployment's query price and accept Parallel bases that end in a route
A proxy request logs through an object built before the router picks the deployment, so a query price on litellm_params never reached Decisions billing. Parallel Search accepts a base ending in /v1/search and shares PARALLEL_AI_API_BASE with Decisions, so Decisions now accepts that base and one ending in /v1/decisions.
* fix(decisions): keep per-query billing and optional confidence on Parallel only
A query price on any other Decisions provider moved that provider off token
billing, and a choice or score answer with no confidence became a 200 with a
null for every provider. Both now apply to Parallel alone, so other providers
bill and fail as they do on main.
A Parallel deployment with explicit zero token rates and a query price read as
free, so every budget check was skipped while it billed per call. It now counts
as priced.
The dashboard key for Parallel is now PARALLEL_AI so a served bundle that
predates the provider still sends parallel_ai as the provider slug, and the
README table lists the provider.
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
README.md | 1 +
litellm/__init__.py | 3 +
litellm/_lazy_imports_registry.py | 2 +
litellm/cost_calculator.py | 9 +-
litellm/decisions/call.py | 10 +-
.../litellm_core_utils/decisions_pricing.py | 12 +
.../llms/base_llm/decisions/transformation.py | 15 +-
.../llms/openai/decisions/transformation.py | 8 +-
.../llms/parallel_ai/decisions/__init__.py | 0
.../parallel_ai/decisions/transformation.py | 89 ++++
...odel_prices_and_context_window_backup.json | 13 +
.../provider_endpoints_support_backup.json | 2 +-
litellm/proxy/_lazy_openapi_snapshot.json | 76 +++-
litellm/proxy/auth/auth_checks.py | 17 +-
.../provider_create_fields.json | 28 ++
.../complexity_router/complexity_router.py | 8 +-
.../complexity_router/jev_classifier.py | 4 +-
litellm/types/decisions.py | 12 +-
litellm/types/utils.py | 1 +
litellm/utils.py | 9 +-
model_prices_and_context_window.json | 13 +
provider_endpoints_support.json | 2 +-
tests/integration/proxy_config.yaml | 5 +
.../decisions/bases/parallel_ai.py | 257 +++++++++++
.../basic/test_decisions_basic_parallel_ai.py | 11 +
.../test_decisions_pricing.py | 60 +++
.../test_base_decisions_transformation.py | 45 ++
.../test_openai_decisions_transformation.py | 37 +-
.../llms/parallel_ai/decisions/__init__.py | 0
.../decisions/test_transformation.py | 423 ++++++++++++++++++
.../public_endpoints/test_public_endpoints.py | 1 +
.../test_zero_cost_model_budget_bypass.py | 23 +
.../router_strategy/test_complexity_router.py | 46 ++
.../ResponseView.integration.test.tsx | 14 +
.../components/systemOneUI/ResponseView.tsx | 4 +-
.../systemOneUI/lib/openAIDecisions.ts | 4 +-
.../components/systemOneUI/lib/schemas.ts | 4 +-
.../components/provider_info_helpers.test.tsx | 16 +
.../src/components/provider_info_helpers.tsx | 5 +
ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +-
40 files changed, 1242 insertions(+), 55 deletions(-)
create mode 100644 litellm/llms/parallel_ai/decisions/__init__.py
create mode 100644 litellm/llms/parallel_ai/decisions/transformation.py
create mode 100644 tests/integration/translation/decisions/bases/parallel_ai.py
create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py
create mode 100644 tests/unit/llms/parallel_ai/decisions/__init__.py
create mode 100644 tests/unit/llms/parallel_ai/decisions/test_transformation.py
diff --git a/README.md b/README.md
index 3b5710fa0c5..c4aeee80fb4 100644
--- a/README.md
+++ b/README.md
@@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
+| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) | | | | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | |
diff --git a/litellm/__init__.py b/litellm/__init__.py
index baee4e9f7ba..041e3fc7837 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -1754,6 +1754,9 @@ if TYPE_CHECKING:
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)
+ from .llms.parallel_ai.decisions.transformation import (
+ ParallelAIDecisionsConfig as ParallelAIDecisionsConfig,
+ )
from .llms.openrouter.decisions.transformation import (
OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
)
diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py
index c9949613303..65074093dbb 100644
--- a/litellm/_lazy_imports_registry.py
+++ b/litellm/_lazy_imports_registry.py
@@ -163,6 +163,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
+ "ParallelAIDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
@@ -740,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
+ "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 3c028278b2d..ef89a5ca8e7 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -25,7 +25,11 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
-from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
+from litellm.litellm_core_utils.decisions_pricing import (
+ decisions_pricing_fields,
+ decisions_request_price,
+ is_decisions_call_type,
+)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
@@ -605,6 +609,9 @@ def cost_per_token(
decisions_overrides: Final = decisions_pricing_fields(
litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
)
+ request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides)
+ if request_price is not None:
+ return request_price, 0.0
if decisions_overrides:
return generic_cost_per_token(
model=model,
diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py
index 871fa8227ea..9c0e51e14ac 100644
--- a/litellm/decisions/call.py
+++ b/litellm/decisions/call.py
@@ -13,6 +13,7 @@ from typing_extensions import assert_never
import litellm
from litellm.cost_calculator import cost_map_key_prices_anything
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
+from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@@ -555,7 +556,7 @@ def prepare_call(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or call_api_base, dynamic_api_key or api_key
)
- except ValueError as error:
+ except (ValueError, httpx.InvalidURL) as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
if resolved_api_base is None:
raise litellm.BadRequestError(
@@ -613,6 +614,9 @@ def prepare_call(
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
+ "input_cost_per_query": (
+ kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
+ ),
},
custom_llm_provider=provider,
)
@@ -787,7 +791,9 @@ def _formatted_response(
case DecisionsRequestBody():
return ir_to_systemone_response(caller_response, caller_request)
case OpenAIDecisionRequestBody():
- return ir_to_openai_response(caller_response, caller_request, call.requested_model)
+ return ir_to_openai_response(
+ caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
+ )
case _:
assert_never(call.request)
diff --git a/litellm/litellm_core_utils/decisions_pricing.py b/litellm/litellm_core_utils/decisions_pricing.py
index 9199a8097cc..accd315da23 100644
--- a/litellm/litellm_core_utils/decisions_pricing.py
+++ b/litellm/litellm_core_utils/decisions_pricing.py
@@ -12,6 +12,7 @@ from typing import Final
DECISIONS_PRICING_SUFFIX: Final = "_decisions"
DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
+QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
def is_decisions_call_type(call_type: str) -> bool:
@@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
}
+def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
+ if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
+ return None
+ rates: Final = {**model_info, **overrides}
+ request_price: Final = rates.get("input_cost_per_query")
+ if not isinstance(request_price, (int, float)):
+ return None
+ bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token"))
+ return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price)
+
+
def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]:
"""Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields."""
plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)}
diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py
index c562c8d0f04..254783edf39 100644
--- a/litellm/llms/base_llm/decisions/transformation.py
+++ b/litellm/llms/base_llm/decisions/transformation.py
@@ -431,6 +431,10 @@ class DecisionsResponseValidationError(BaseLLMException):
pass
+def _lacks_confidence(answer: DecisionsIRAnswer) -> bool:
+ return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None
+
+
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
return systemone_response_to_ir(
_SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request
@@ -534,6 +538,8 @@ class BaseDecisionsConfig(ABC):
max_total_image_bytes: int | None = None
image_media_types: frozenset[str] | None = None
inlines_remote_images: bool = False
+ openai_evidence_fields: frozenset[str] = frozenset()
+ confidence_optional: bool = False
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
)
@@ -682,13 +688,20 @@ class BaseDecisionsConfig(ABC):
) -> DecisionsIRResponse:
try:
payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content)
- return self.parse_response(payload, request)
+ response: Final = self.parse_response(payload, request)
except ValidationError as error:
raise DecisionsResponseValidationError(
status_code=500,
message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
f"{error.title} {_validation_summary(error)}",
) from error
+ if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers):
+ raise DecisionsResponseValidationError(
+ status_code=500,
+ message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
+ "a choice or score answer has no confidence",
+ )
+ return response
def get_error_class(
self,
diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py
index 369b6959411..5aa72579d6a 100644
--- a/litellm/llms/openai/decisions/transformation.py
+++ b/litellm/llms/openai/decisions/transformation.py
@@ -307,10 +307,13 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) ->
def ir_to_openai_response(
- response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str
+ response: DecisionsIRResponse,
+ request: DecisionsIRRequest,
+ requested_model: str,
+ evidence_fields: frozenset[str] = frozenset(),
) -> OpenAIDecisionResponse:
usage: Final = response.usage or DecisionsIRUsage()
- return OpenAIDecisionResponse(
+ formatted: Final = OpenAIDecisionResponse(
model=response.model or requested_model,
answers=tuple(
_openai_answer(question, answer)
@@ -327,6 +330,7 @@ def ir_to_openai_response(
total_tokens=usage.input_tokens + usage.output_tokens,
),
)
+ return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()})
class OpenAIDecisionsConfig(BaseDecisionsConfig):
diff --git a/litellm/llms/parallel_ai/decisions/__init__.py b/litellm/llms/parallel_ai/decisions/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py
new file mode 100644
index 00000000000..34d03047301
--- /dev/null
+++ b/litellm/llms/parallel_ai/decisions/transformation.py
@@ -0,0 +1,89 @@
+from collections.abc import Mapping
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, ConfigDict
+
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, ir_to_systemone_request
+from litellm.types.decisions import (
+ DecisionsIRRequest,
+ DecisionsIRResponse,
+ UnsupportedDecisionsRequest,
+)
+
+
+class _ParallelResponse(BaseModel):
+ model_config = ConfigDict(extra="allow", frozen=True)
+
+ mode: str
+ answers: Mapping[str, Mapping[str, object]]
+
+
+class ParallelAIDecisionsConfig(BaseDecisionsConfig):
+ path = "/v1/decisions"
+ api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY")
+ api_base_env = ("PARALLEL_AI_API_BASE",)
+ confidence_optional = True
+ openai_evidence_fields = frozenset({"citations", "decision_id", "mode"})
+
+ def get_default_api_base(self) -> str:
+ return "https://api.parallel.ai"
+
+ def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
+ configured_base: Final = self.resolve_api_base(None)
+ resolved_base: Final = api_base or configured_base
+ destination: Final = self._decisions_url(resolved_base)
+ if api_key:
+ return resolved_base, api_key
+ if destination not in {self._decisions_url(None), self._decisions_url(configured_base)}:
+ raise ValueError(
+ "Parallel Decisions requires an explicit api_key for a custom api_base unless "
+ "the destination is configured in PARALLEL_AI_API_BASE"
+ )
+ return resolved_base, self.resolve_api_key(None)
+
+ def get_complete_url(self, api_base: str, model: str) -> str:
+ return super().get_complete_url(api_base.rstrip("/").removesuffix("/v1/search").removesuffix(self.path), model)
+
+ def _decisions_url(self, api_base: str | None) -> httpx.URL:
+ return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision"))
+
+ def validate_environment(
+ self,
+ headers: Mapping[str, str],
+ model: str,
+ api_key: str | None,
+ litellm_params: Mapping[str, object],
+ ) -> Mapping[str, str]:
+ return {
+ **{
+ name: value
+ for name, value in headers.items()
+ if name.lower() not in {"authorization", "x-api-key", "content-type"}
+ },
+ **({"x-api-key": api_key} if api_key is not None else {}),
+ "Content-Type": "application/json",
+ }
+
+ def transform_decisions_request(
+ self, model: str, request: DecisionsIRRequest, custom_llm_provider: str
+ ) -> Mapping[str, object] | UnsupportedDecisionsRequest:
+ if model != "decision":
+ return UnsupportedDecisionsRequest(reason="Parallel Decisions supports the decision model")
+ if len(request.questions) > 100:
+ return UnsupportedDecisionsRequest(reason="Parallel Decisions supports at most 100 questions")
+ body: Final = ir_to_systemone_request(model, request)
+ if isinstance(body, UnsupportedDecisionsRequest):
+ return body
+ return {"mode": "fast", **{key: value for key, value in body.items() if key != "model"}}
+
+ def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+ response: Final = _ParallelResponse.model_validate(payload)
+ return super().parse_response(
+ {
+ **response.model_dump(),
+ "model": "decision",
+ "answers": {key: answer for key, answer in response.answers.items() if answer.get("type") != "refusal"},
+ },
+ request,
+ )
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 032b7d17423..75364a382f7 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -46412,6 +46412,19 @@
"supports_response_schema": true,
"supports_tool_choice": false
},
+ "parallel_ai/decision": {
+ "input_cost_per_query": 0.001,
+ "input_cost_per_token": 0.0,
+ "output_cost_per_token": 0.0,
+ "litellm_provider": "parallel_ai",
+ "max_input_tokens": 32000,
+ "mode": "evaluation",
+ "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api",
+ "supported_endpoints": [
+ "/v1/decisions",
+ "/v1/systemone"
+ ]
+ },
"parallel_ai/search": {
"input_cost_per_query": 0.005,
"litellm_provider": "parallel_ai",
diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json
index 700a35edb06..40ea9d15ff4 100644
--- a/litellm/provider_endpoints_support_backup.json
+++ b/litellm/provider_endpoints_support_backup.json
@@ -3158,7 +3158,7 @@
"batches": false,
"messages": false,
"interactions": false,
- "systemone": false,
+ "systemone": true,
"a2a": false,
"generateContent": false
}
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index 5a80583b207..ba6494c3bda 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -10237,8 +10237,15 @@
"type": "string"
},
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"probabilities": {
"additionalProperties": {
@@ -10256,7 +10263,6 @@
"required": [
"type",
"choice",
- "confidence",
"probabilities"
],
"title": "ChoiceAnswer",
@@ -10687,8 +10693,15 @@
"title": "Choice"
},
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"name": {
"anyOf": [
@@ -10718,8 +10731,7 @@
"required": [
"name",
"choice",
- "probabilities",
- "confidence"
+ "probabilities"
],
"title": "OpenAIChoiceAnswer",
"type": "object"
@@ -10924,8 +10936,15 @@
"OpenAIScoreAnswer": {
"properties": {
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"name": {
"anyOf": [
@@ -10959,8 +10978,7 @@
"required": [
"name",
"score",
- "probabilities",
- "confidence"
+ "probabilities"
],
"title": "OpenAIScoreAnswer",
"type": "object"
@@ -10992,8 +11010,15 @@
"additionalProperties": true,
"properties": {
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"legend": {
"additionalProperties": {
@@ -11034,7 +11059,6 @@
"required": [
"type",
"score",
- "confidence",
"legend",
"probabilities"
],
@@ -23542,8 +23566,15 @@
"type": "string"
},
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"probabilities": {
"additionalProperties": {
@@ -23561,7 +23592,6 @@
"required": [
"type",
"choice",
- "confidence",
"probabilities"
],
"title": "ChoiceAnswer",
@@ -29400,8 +29430,15 @@
"additionalProperties": true,
"properties": {
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"legend": {
"additionalProperties": {
@@ -29442,7 +29479,6 @@
"required": [
"type",
"score",
- "confidence",
"legend",
"probabilities"
],
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index fdc8aa82268..9d98e2c9973 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -43,7 +43,9 @@ from litellm.constants import (
REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
TAG_REGISTRY_MAX_SIZE,
)
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.dd_tracing import tracer
+from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, decisions_request_price
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.tool_fields import DropPolicy
@@ -730,7 +732,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -
def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
- """Whether any of the deployments carries an explicit positive per-token price.
+ """Whether any of the deployments carries an explicit positive per-token or per-query price.
The group's price is read from the deployments named after the routed group and from
that group's own alias target, while an alias chain is served from the deployments named
@@ -741,14 +743,21 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
if model_id is None:
continue
- raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
- if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost(
- raw_entry.get("output_cost_per_token")
+ raw_entry = as_str_mapping(litellm.model_cost.get(model_id)) or _EMPTY_COST_ENTRY
+ if (
+ _is_positive_cost(raw_entry.get("input_cost_per_token"))
+ or _is_positive_cost(raw_entry.get("output_cost_per_token"))
+ or _is_positive_cost(_deployment_query_price(deployment["litellm_params"], raw_entry))
):
return True
return False
+def _deployment_query_price(litellm_params: Mapping[str, object], cost_entry: Mapping[str, object]) -> float | None:
+ provider: Final = litellm_params.get("custom_llm_provider") or str(litellm_params.get("model")).partition("/")[0]
+ return decisions_request_price({**cost_entry, "litellm_provider": provider}, decisions_pricing_fields(cost_entry))
+
+
_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})
diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json
index 044b8a1de6e..aa7554c4a5a 100644
--- a/litellm/proxy/public_endpoints/provider_create_fields.json
+++ b/litellm/proxy/public_endpoints/provider_create_fields.json
@@ -2641,6 +2641,34 @@
],
"default_model_placeholder": "nace/drex-latest"
},
+ {
+ "provider": "PARALLEL_AI",
+ "provider_display_name": "Parallel AI",
+ "litellm_provider": "parallel_ai",
+ "credential_fields": [
+ {
+ "key": "api_base",
+ "label": "API Base",
+ "placeholder": "https://api.parallel.ai",
+ "tooltip": null,
+ "required": false,
+ "field_type": "text",
+ "options": null,
+ "default_value": null
+ },
+ {
+ "key": "api_key",
+ "label": "API Key",
+ "placeholder": null,
+ "tooltip": null,
+ "required": true,
+ "field_type": "password",
+ "options": null,
+ "default_value": null
+ }
+ ],
+ "default_model_placeholder": "parallel_ai/decision"
+ },
{
"provider": "NEBIUS",
"provider_display_name": "Nebius",
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index 4f4ed76c170..05dd4706b62 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -1219,9 +1219,11 @@ def _with_classifier_forecast(
forecasted_decision: Final[StandardLoggingRoutingDecision] = {
**with_input,
"classifier_probabilities": outcome.jev_verdict.probabilities,
- "classifier_confidence": outcome.jev_verdict.confidence,
}
- return forecasted_decision
+ confidence: Final = outcome.jev_verdict.confidence
+ return (
+ forecasted_decision if confidence is None else {**forecasted_decision, "classifier_confidence": confidence}
+ )
if outcome.llm_v2_forecast is not None:
return _with_llm_v2_forecast(with_input, outcome.llm_v2_forecast)
forecast: Final = outcome.capability_forecast
@@ -2468,7 +2470,7 @@ class ComplexityRouter(CustomLogger):
score=None,
signals=(
f"{config.provider}-classifier:{tier_name}",
- f"{config.provider}-confidence={answer.confidence:.6f}",
+ *(() if answer.confidence is None else (f"{config.provider}-confidence={answer.confidence:.6f}",)),
*(
f"tier-probability:{label}={probability:.6f}"
for label, probability in answer.probabilities.items()
diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py
index bb7d2e91cd3..5cd4f00a941 100644
--- a/litellm/router_strategy/complexity_router/jev_classifier.py
+++ b/litellm/router_strategy/complexity_router/jev_classifier.py
@@ -210,7 +210,7 @@ class JevChoiceAnswer(LiteLLMBaseModel):
type: Literal["choice"]
choice: str
probabilities: Mapping[str, JevProbability]
- confidence: JevProbability
+ confidence: JevProbability | None = None
class JevUsage(LiteLLMBaseModel):
@@ -425,7 +425,7 @@ class HttpJevClassifierClient:
class JevVerdict(NamedTuple):
label: str
probabilities: Mapping[str, float]
- confidence: float
+ confidence: float | None
model: str
cost: float | None
provider: str = "typesafe"
diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py
index 33b2c677d31..3b28225f0ff 100644
--- a/litellm/types/decisions.py
+++ b/litellm/types/decisions.py
@@ -210,7 +210,7 @@ class NoulAnswer(LiteLLMPydanticObjectBase):
class ChoiceAnswer(LiteLLMPydanticObjectBase):
type: Literal["choice"]
choice: str
- confidence: float
+ confidence: float | None = None
probabilities: Mapping[str, float]
model_config = ConfigDict(extra="allow", frozen=True)
@@ -219,7 +219,7 @@ class ChoiceAnswer(LiteLLMPydanticObjectBase):
class ScoreAnswer(LiteLLMPydanticObjectBase):
type: Literal["score"]
score: float
- confidence: float
+ confidence: float | None = None
legend: Mapping[str, DecisionsJSON]
probabilities: Mapping[str, float]
@@ -431,7 +431,7 @@ class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase):
name: str | None
choice: str | bool
probabilities: tuple[OpenAIChoiceProbability, ...]
- confidence: float
+ confidence: float | None = None
model_config = ConfigDict(frozen=True)
@@ -449,7 +449,7 @@ class OpenAIScoreAnswer(LiteLLMPydanticObjectBase):
name: str | None
score: float
probabilities: tuple[OpenAIScoreProbability, ...]
- confidence: float
+ confidence: float | None = None
model_config = ConfigDict(frozen=True)
@@ -595,7 +595,7 @@ class DecisionsIRChoiceProbability:
@dataclass(frozen=True, slots=True)
class DecisionsIRChoiceAnswer:
choice: str | bool
- confidence: float
+ confidence: float | None
probabilities: tuple[DecisionsIRChoiceProbability, ...]
extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA)
@@ -610,7 +610,7 @@ class DecisionsIRScoreProbability:
@dataclass(frozen=True, slots=True)
class DecisionsIRScoreAnswer:
score: float
- confidence: float
+ confidence: float | None
probabilities: tuple[DecisionsIRScoreProbability, ...]
extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA)
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 080ea9848b6..e633e557bb2 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -4580,6 +4580,7 @@ class LlmProviders(str, Enum):
CLF_AI_GATEWAY = "clf_ai_gateway"
PERPLEXITY = "perplexity"
TYPESAFE = "typesafe"
+ PARALLEL_AI = "parallel_ai"
NACE = "nace"
STRANDS_DECIDER = "strands_decider"
LAYA = "laya"
diff --git a/litellm/utils.py b/litellm/utils.py
index 4d3399d1d0c..ddfc38ca677 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -10105,12 +10105,19 @@ class ProviderConfigManager:
return litellm.LayaDecisionsConfig()
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
- if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV):
+ if provider in (
+ LlmProviders.OPENAI,
+ LlmProviders.AZURE,
+ LlmProviders.AZURE_AI,
+ LlmProviders.EU_JEV,
+ LlmProviders.PARALLEL_AI,
+ ):
configs: Final = {
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig,
LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig,
+ LlmProviders.PARALLEL_AI: litellm.ParallelAIDecisionsConfig,
}
return configs[provider]()
return None
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 032b7d17423..75364a382f7 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -46412,6 +46412,19 @@
"supports_response_schema": true,
"supports_tool_choice": false
},
+ "parallel_ai/decision": {
+ "input_cost_per_query": 0.001,
+ "input_cost_per_token": 0.0,
+ "output_cost_per_token": 0.0,
+ "litellm_provider": "parallel_ai",
+ "max_input_tokens": 32000,
+ "mode": "evaluation",
+ "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api",
+ "supported_endpoints": [
+ "/v1/decisions",
+ "/v1/systemone"
+ ]
+ },
"parallel_ai/search": {
"input_cost_per_query": 0.005,
"litellm_provider": "parallel_ai",
diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json
index 700a35edb06..40ea9d15ff4 100644
--- a/provider_endpoints_support.json
+++ b/provider_endpoints_support.json
@@ -3158,7 +3158,7 @@
"batches": false,
"messages": false,
"interactions": false,
- "systemone": false,
+ "systemone": true,
"a2a": false,
"generateContent": false
}
diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml
index 4e2cab56d27..c552e11c22a 100644
--- a/tests/integration/proxy_config.yaml
+++ b/tests/integration/proxy_config.yaml
@@ -587,6 +587,11 @@ model_list:
model: perplexity/pplx-decider-v1-27b
api_base: http://127.0.0.1:8191
api_key: synthetic-perplexity-key
+ - model_name: parallel_ai/decision
+ litellm_params:
+ model: parallel_ai/decision
+ api_base: http://127.0.0.1:8191
+ api_key: synthetic-parallel-key
- model_name: typesafe/jev-1.13.0
litellm_params:
model: typesafe/jev-1.13.0
diff --git a/tests/integration/translation/decisions/bases/parallel_ai.py b/tests/integration/translation/decisions/bases/parallel_ai.py
new file mode 100644
index 00000000000..14e9cd6fc43
--- /dev/null
+++ b/tests/integration/translation/decisions/bases/parallel_ai.py
@@ -0,0 +1,257 @@
+from typing import Final
+
+from integration.translation.case import TranslationTestCase
+from pydantic import JsonValue
+
+"""Provider request and reply shape from https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions. Mock reply captured live on 2026-10-31.
+"""
+_CITATIONS: Final[list[JsonValue]] = [
+ {
+ "url": "https://access.redhat.com/solutions/7108129",
+ "title": 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg. - Red Hat Customer Portal',
+ "excerpts": [
+ 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg.\n\nSolution Verified - Updated 2026-06-18T01:07:40+00:00 -\nEnglish\n* No translations currently exist.\n\nIssue\n\n* NFS client hangs while returning delegations.\n* Below delegation recall errors may be reported in dmesg:\nNFS: nfs4_handle_delegation_recall_error: unhandled error -512\nNFS: nfs4_handle_delegation_recall_error: unhandled error -10024\n* We have a hanging process, it\'s using data ontop of a mounted NFS export. The fcntl() system call issued by a user process has not returned, The user process processing does not finish - usually it finishes within 2 minutes.\n\nEnvironment\n\n* Red Hat Enterprise Linux (RHEL) 8, all minor releases\n* kernel:\n* rhel8.4: kernels before 4.18.0-305.194.1.el8_4 affected\n* rhel8.10: kernels before 4.18.0-553.37.1.el8_10 affected\n* NFSv4'
+ ],
+ },
+ {
+ "url": "https://github.com/moby/buildkit/issues/2804",
+ "title": "GitHub cache exporter (`type=gha`) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. · Issue #2804 · moby/buildkit · GitHub",
+ "excerpts": [
+ 'Copy link\nOpen\nOpen\nGitHub cache exporter ( type=gha ) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. #2804\nCopy link\n\nDescription\n\n@bendavies\nbendavies\nopened on Apr 13, 2022\nIssue body actions\n\nBehaviour\n\nCache operations using type=gha will start hanging after a relavatively low number of cache reads/writes.\nWe are a small team (5 devs) and are hitting these limits constantly on our main repository. Not exactly high throughput.\n\n...\n\nCache reads and writes go towards saturating the limit, and both are affected by this throttling.\n@dhadka mentions there is a limit of about "150 cache requests in 5 minutes", and seems about right: #2276 (comment)\n\nExpected behaviour\n\nI don\'t know, but the current behaviour of hanging is not great. there is no feedback as to what is hanging resulting in the user just being confused.'
+ ],
+ },
+ {
+ "url": "https://github.com/huginn/huginn/issues/2963",
+ "title": "WebsiteAgent hangs on a specific url, never finishes job or times out · Issue #2963 · huginn/huginn · GitHub",
+ "excerpts": [
+ 'WebsiteAgent hangs on a specific url, never finishes job or times out\n\n* Page: GitHub issue\n* URL: https://github.com/huginn/huginn/issues/2963\n* State: open\n* Author: BoKKeR\n* Created: 2021-03-14T16:42:01Z\n* Updated: 2021-03-16T19:09:24Z\n* Repository: huginn/huginn\n* Number: #2963\n* Comments: 1\nI know it might be controversial but I am trying to get a new GPU for a while. Wanted to setup a monitor for when the stock on amd.com changes. But the site just won\'t load. The WebsiteAgent gets stuck in\nFetching https://www.amd.com/de/direct-buy/se\nThe only way to stop it is by stopping the container. As I am running the service in a container I cant find any of the debug log locations. Can anyone else try if they are encountering the same error?\n{\n "debug": "true",\n "expected_update_period_in_days": "2",\n "url": "https://www.amd.com/de/direct-buy/se",\n "type": "html",'
+ ],
+ },
+ {
+ "url": "https://github.com/mozilla-firefox/firefox-crb/blob/main/docs/bug-mgmt/guides/severity.md",
+ "title": "firefox-crb/docs/bug-mgmt/guides/severity.md at main · mozilla-firefox/firefox-crb · GitHub",
+ "excerpts": [
+ "We use the severity field in Bugzilla to indicate the scope of a bug's effect on Firefox.\nThe field is display alongside the bug's priority.\n\nValues\n\nSeverity levels and their definitions are enumerated at https://wiki.mozilla.org/BMO/UserGuide/BugFields#bug_severity.\nBy default, new bugs have a severity of --.\n\nExamples of S1 bugs\n\n* WebExtensions disabled for all users\n* Web search not working from URL bar\n* Crashes with data loss\n\nExamples of S2 bugs\n\nBugs that could reasonably be expected to cause a Firefox user to switch browsers, either because the severity is bad enough, or the frequency of occurrence is high enough.\n* Bug 1640441 - Slack hangs indefinitely in a onResize loop"
+ ],
+ },
+ {
+ "url": "https://github.com/n0-computer/iroh/issues/4604",
+ "title": "Endpoint::close hangs forever if the tokio runtime gets dropped while the connection is open · Issue #4604 · n0-computer/iroh · GitHub",
+ "excerpts": [
+ "@alpergundogdu\nalpergundogdu\nopened on Oct 14, 2026\nIssue body actions\nDescribe the bug\nAfter opening a connection from inside a short-lived tokio runtime, and dropping the runtime while the connection is still open, Endpoint::close() never returns.\nIn my repro it hangs reliably; when I keep the runtime open, it closes instantly.\nLocal repro/test (has the control tests too): https://github.com/decdn/iroh-tests/blob/main/tests/bug_endpoint_close_after_runtime_drop.rs\n\n...\n\nIf spawning on the caller's runtime is intended, a note on Endpoint::connect saying the driver lives there would have saved us a lot of time. (Maybe it's an expectation/documentation issue?)\nIroh\nVersion: iroh 1.3.0, noq 1.3.0\nEndpoint configuration: bound to 127.0.0.1, no relays.\nPlatform(s)\nDesktop:\n\n...\n\nSorry @flub , that was unclear: the endpoint's runtime stays alive, and close() is called from it. Only a separate short-lived runtime that runs connect() gets dropped."
+ ],
+ },
+ {
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1640441",
+ "title": "1640441 - Slack hangs indefinitely in a onResize loop",
+ "excerpts": [
+ "Closed Bug 1640441 Opened 6 years ago Closed 6 years ago\n\nSlack hangs indefinitely in a on Resize loop\n\nCategories\n\n(Web Compatibility :: Site Reports, defect, P1)\n\nProduct:\nWeb Compatibility ▾\nComponent:\nSite Reports ▾\nType:\ndefect\nPriority:\nP1 Severity:\nS2\n\nTracking\n\n(Not tracked)\n\nStatus:\nRESOLVED FIXED\n\nPeople\n\n(Reporter: Harald, Assigned: emilio)\n\nReferences\n\n(Depends on 1 open bug)\n\nDetails\n\n(Keywords: enterprise, perf, webcompat:site-wait)\n\nAttachments\n\n(1 file, 1 obsolete file)\n\n|relevant unminified code.\n6 years ago Emilio Cobos Álvarez [:emilio]\n* 13.50 KB,\n\n...\n\n* Hover over the link\n* ... tab immediately becomes unresponsive ...\nThis may well be a bug in slack, but IMO it's still on us to try and fix what's going on. When it happens, it affects not only Slack but my entire Firefox session. This is a serious usability issue -- I'm now at the point of restarting Firefox several times a day because of this bug, if"
+ ],
+ },
+ {
+ "url": "https://www.ibm.com/support/pages/cognos-business-intelligence-1022-fix-lists",
+ "title": "Cognos Business Intelligence 10.2.2 Fix Lists",
+ "excerpts": [
+ 'IBM Support \n\nCognos Business Intelligence 10.2.2 Fix Lists\n\nRelease Notes\n\nAbstract\n\nA comprehensive list of defect corrections for major releases, refresh packs and fix packs of Cognos Business Intelligence 10.2.2.\nDetails of the APARs listed below can be accessed by clicking the link for the APAR number. If you have questions about a particular defect, please contact Customer Support.\n\n...\n\n* PI05143 | MULTIPLE VALUES IN CONTEXT FILTERS CAUSE A CROSSTAB REPORT TO SHOW DASH "-" INSTEAD OF CORRECT VALUES\n* PI05166 | QE-DEF-0359 The query contains a reference to at least one object that does not exist\n* PI05233 | Export deployments from 8.4.1 with export folder or package tasks in Content Administration contain unknown content steps\n\n...\n\n* PI08523 | Progressive Chart - Incorrect Colours when "Reverse Catagory Order" = True\n* PI08585 | Cognos hangs when starting against a Content Store from previous version (upgrade) and using SSL gateway UR'
+ ],
+ },
+ {
+ "url": "https://josm.eu/ticket/8175",
+ "title": "\n #8175 ([PATCH] Cadastre grab query sometimes hangs completely)\n – JOSM\n\n ",
+ "excerpts": [
+ 'Opened 14 years ago\nClosed 14 years ago\nLast modified 14 years ago\n\n#8175 closed defect ( fixed )\n\n[PATCH] Cadastre grab query sometimes hangs completely\n\n* Reported by: | Don-vip | Owned by: | pieren\n* Priority: | normal | Milestone: | \n* Component: | Plugin cadastre-fr | Version: | \n* Keywords: | | Cc: | \n\nDescription\n\nDuring cadastre grab, the HTTP connection with the WMS server sometimes completely hangs (the progress dialog is frozen and cancel button does not work).\nThe only solution is to restart JOSM to have a clean state and grab the cadastre again. We had a similar problem with history request ( #7640 ) when dealing with the API and solved it in [r5324](https://josm.eu/changeset/5324/josm "fix #7640 - History query will not cancel or timeout\n\n...\n\nby Don-vip , 14 years ago\n\n* Attachment: | 8175.patch \u200b added\n\ncomment:1 by Don-vip , 14 years ago'
+ ],
+ },
+ {
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1749908",
+ "title": "1749908 - Infinite loop in HTTP3 hangs socket thread - Bugzilla",
+ "excerpts": [
+ "Closed Bug 1749908 Opened 4 years ago Closed 4 years ago\n\nInfinite loop in HTTP3 hangs socket thread\n\nCategories\n\n(Core :: Networking, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking ▾\nPlatform:\nDesktop\nAll\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nVERIFIED DUPLICATE of bug 1749910\n\nPeople\n\n(Reporter: heftig, Unassigned)\n\nDetails\n\nBottom ↓\nTags ▾\nTimeline ▾\n| |Jan Alexander Steffens [:heftig]\n* Reporter | \n* ### Description\n•\n* 4 years ago\n* \nWhile browsing, pages suddenly stopped loading.\n\n...\n\nIt's not related to a specific version, we're getting reports ESR is even affected. Suspicion is around some long existing HTTP3 bug that's being triggered by an external service updating.\n\n...\n\nI agree it is not the root cause but it certainly caused a massive problem for many users of Firefox, the majority of whom are opted-in by default. If they had not been, they would not have been affected."
+ ],
+ },
+ {
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1748414",
+ "title": "1748414 - Firefox 94 sometimes hangs on TLSv1.3 requests",
+ "excerpts": [
+ "Closed Bug 1748414 Opened 4 years ago Closed 4 years ago\n\nFirefox 94 sometimes hangs on TLSv1 .3 requests\n\nCategories\n\n(Core :: Networking: HTTP, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking: HTTP ▾\nVersion:\nFirefox 94\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nRESOLVED DUPLICATE of bug 1740856\n\nPeople\n\n(Reporter: code, Unassigned)\n\nDetails\n\nBottom ↓\n\n...\n\nSteps to reproduce:\nIn the beginning of November, colleagues started to experience hangs of 1 to 1.5 minutes when working with our own web app. We use latest official releases for Windows, Ubuntu and FreeBSD. The problems exist for all plattforms and very likely started with Firefox 94.\nActual results:\nSometimes while interacting normally with our web app, a request hangs for a long time (up to 1.5 minutes; but usually succeeds). This typically happens on POST requests."
+ ],
+ },
+]
+DECISION_TEST_CASE: Final = TranslationTestCase(
+ scenario="basic",
+ litellm_endpoint="/v1/decisions",
+ litellm_request={
+ "model": "parallel_ai/decision",
+ "input": "Ticket (billing): The export job hangs at 99% and never finishes",
+ "questions": [
+ {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
+ {
+ "type": "choice",
+ "name": "severity",
+ "instructions": "How severe is it?",
+ "choices": [
+ {"value": "low", "description": "cosmetic"},
+ {"value": "high", "description": "blocks users"},
+ ],
+ },
+ {
+ "type": "score",
+ "name": "confidence",
+ "instructions": "How sure are you?",
+ "levels": [{"label": "unsure"}, {"label": "sure"}],
+ },
+ ],
+ "cache": {"no-cache": True},
+ },
+ expected_provider_endpoint="/v1/decisions",
+ expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"},
+ expected_provider_request={
+ "mode": "fast",
+ "state": "Ticket (billing): The export job hangs at 99% and never finishes",
+ "questions": {
+ "defect": {"type": "noul", "instructions": "Is this a defect?"},
+ "severity": {
+ "type": "choice",
+ "instructions": "How severe is it?",
+ "criteria": {"low": "cosmetic", "high": "blocks users"},
+ },
+ "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
+ },
+ },
+ mock_provider_response={
+ "mode": "fast",
+ "answers": {
+ "defect": {"type": "noul", "noul": 0.6442559258537284},
+ "severity": {
+ "type": "choice",
+ "choice": "high",
+ "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014},
+ "confidence": 0.6766070979468028,
+ },
+ "confidence": {
+ "type": "score",
+ "score": 0.25195637565890455,
+ "legend": {"0": "unsure", "1": "sure"},
+ "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455},
+ "confidence": 0.4960872486821909,
+ },
+ },
+ "usage": {"input_tokens": 3166, "output_tokens": 0},
+ "citations": _CITATIONS,
+ "decision_id": "decisions_9fd99def328ed16256c11df297dea899",
+ },
+ expected_litellm_response={
+ "model": "decision",
+ "answers": [
+ {"type": "predicate", "name": "defect", "probability": 0.6442559258537284},
+ {
+ "type": "choice",
+ "name": "severity",
+ "choice": "high",
+ "probabilities": [
+ {"value": "low", "probability": 0.1616964510265985},
+ {"value": "high", "probability": 0.8383035489734014},
+ ],
+ "confidence": 0.6766070979468028,
+ },
+ {
+ "type": "score",
+ "name": "confidence",
+ "score": 0.25195637565890455,
+ "probabilities": [
+ {"value": 0, "label": "unsure", "probability": 0.7480436243410955},
+ {"value": 1, "label": "sure", "probability": 0.25195637565890455},
+ ],
+ "confidence": 0.4960872486821909,
+ },
+ ],
+ "usage": {
+ "input_tokens": 3166,
+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
+ "output_tokens": 0,
+ "output_tokens_details": {"reasoning_tokens": 0},
+ "total_tokens": 3166,
+ },
+ "mode": "fast",
+ "citations": _CITATIONS,
+ "decision_id": "decisions_9fd99def328ed16256c11df297dea899",
+ },
+)
+DECISION_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
+ scenario="systemone",
+ litellm_endpoint="/v1/systemone",
+ litellm_request={
+ "model": "parallel_ai/decision",
+ "state": "Ticket (billing): The export job hangs at 99% and never finishes",
+ "questions": {
+ "defect": {"type": "noul", "instructions": "Is this a defect?"},
+ "severity": {
+ "type": "choice",
+ "instructions": "How severe is it?",
+ "criteria": {"low": "cosmetic", "high": "blocks users"},
+ },
+ "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
+ },
+ "cache": {"no-cache": True},
+ },
+ expected_provider_endpoint="/v1/decisions",
+ expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"},
+ expected_provider_request={
+ "mode": "fast",
+ "state": "Ticket (billing): The export job hangs at 99% and never finishes",
+ "questions": {
+ "defect": {"type": "noul", "instructions": "Is this a defect?"},
+ "severity": {
+ "type": "choice",
+ "instructions": "How severe is it?",
+ "criteria": {"low": "cosmetic", "high": "blocks users"},
+ },
+ "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
+ },
+ },
+ mock_provider_response={
+ "mode": "fast",
+ "answers": {
+ "defect": {"type": "noul", "noul": 0.6442559258537284},
+ "severity": {
+ "type": "choice",
+ "choice": "high",
+ "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014},
+ "confidence": 0.6766070979468028,
+ },
+ "confidence": {
+ "type": "score",
+ "score": 0.25195637565890455,
+ "legend": {"0": "unsure", "1": "sure"},
+ "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455},
+ "confidence": 0.4960872486821909,
+ },
+ },
+ "usage": {"input_tokens": 3166, "output_tokens": 0},
+ "citations": _CITATIONS,
+ "decision_id": "decisions_9fd99def328ed16256c11df297dea899",
+ },
+ expected_litellm_response={
+ "model": "decision",
+ "answers": {
+ "defect": {"type": "noul", "noul": 0.6442559258537284},
+ "severity": {
+ "type": "choice",
+ "choice": "high",
+ "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014},
+ "confidence": 0.6766070979468028,
+ },
+ "confidence": {
+ "type": "score",
+ "score": 0.25195637565890455,
+ "legend": {"0": "unsure", "1": "sure"},
+ "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455},
+ "confidence": 0.4960872486821909,
+ },
+ },
+ "usage": {"input_tokens": 3166, "output_tokens": 0},
+ "mode": "fast",
+ "citations": _CITATIONS,
+ "decision_id": "decisions_9fd99def328ed16256c11df297dea899",
+ },
+)
diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py
new file mode 100644
index 00000000000..47eb057323b
--- /dev/null
+++ b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py
@@ -0,0 +1,11 @@
+import pytest
+from integration._support.client import Gateway
+from integration._support.provider import SharedProvider
+from integration.translation.case import TranslationTestCase
+from integration.translation.decisions.bases.parallel_ai import DECISION_SYSTEMONE_TEST_CASE, DECISION_TEST_CASE
+from integration.translation.runner import assert_translation
+
+
+@pytest.mark.parametrize("case", [DECISION_TEST_CASE, DECISION_SYSTEMONE_TEST_CASE], ids=lambda case: case.id)
+def test_decisions_basic_parallel_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
+ assert_translation(case, gateway, provider)
diff --git a/tests/unit/litellm_core_utils/test_decisions_pricing.py b/tests/unit/litellm_core_utils/test_decisions_pricing.py
index e9814b980d1..d9603230319 100644
--- a/tests/unit/litellm_core_utils/test_decisions_pricing.py
+++ b/tests/unit/litellm_core_utils/test_decisions_pricing.py
@@ -62,6 +62,66 @@ def test_apply_decisions_pricing_leaves_rows_without_decisions_fields_unchanged(
assert apply_decisions_pricing(plain_row) == plain_row
+@pytest.mark.parametrize("rate", [0.0, 0.002])
+def test_flat_decisions_rate_overrides_catalog_without_changing_chat(
+ monkeypatch: pytest.MonkeyPatch, rate: float
+) -> None:
+ model: Final = "parallel_ai/decision"
+ monkeypatch.setitem(
+ litellm.model_cost,
+ model,
+ {**_ROW, "litellm_provider": "parallel_ai", "input_cost_per_query_decisions": rate},
+ )
+ assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (rate, 0.0)
+ assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="completion") == pytest.approx(
+ (100 * 1e-07, 0.0)
+ )
+
+
+@pytest.mark.parametrize("query_field", ["input_cost_per_query", "input_cost_per_query_decisions"])
+def test_query_price_on_a_provider_that_bills_tokens_stays_unbilled(
+ monkeypatch: pytest.MonkeyPatch, query_field: str
+) -> None:
+ model: Final = "fireworks_ai/accounts/example/models/query-priced"
+ monkeypatch.setitem(
+ litellm.model_cost,
+ model,
+ {query_field: 0.5, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
+ | {"litellm_provider": "fireworks_ai", "mode": "evaluation"},
+ )
+ assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (0.0, 0.0)
+
+
+@pytest.mark.parametrize("call_type", ["adecisions", "asystemone"])
+@pytest.mark.parametrize(
+ ("token_rates", "expected_costs"),
+ [
+ ({"input_cost_per_token": 1e-05, "output_cost_per_token": 0.0}, (100 * 1e-05, 0.0)),
+ ({"input_cost_per_token": 0.0, "output_cost_per_token": 1e-05}, (0.0, 10 * 1e-05)),
+ ],
+)
+def test_plain_query_price_leaves_a_token_priced_row_billing_tokens(
+ monkeypatch: pytest.MonkeyPatch,
+ call_type: str,
+ token_rates: dict[str, float],
+ expected_costs: tuple[float, float],
+) -> None:
+ model: Final = "fireworks_ai/accounts/example/models/priced-both"
+ monkeypatch.setitem(
+ litellm.model_cost,
+ model,
+ {"input_cost_per_query": 0.5, **token_rates, "litellm_provider": "fireworks_ai", "mode": "evaluation"},
+ )
+
+ assert cost_calculator.cost_per_token(
+ model=model,
+ custom_llm_provider="fireworks_ai",
+ prompt_tokens=100,
+ completion_tokens=10,
+ call_type=cast(CallTypesLiteral, call_type),
+ ) == pytest.approx(expected_costs)
+
+
_LUNA_USAGE: Final = Usage(
prompt_tokens=100,
completion_tokens=10,
diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py
index 041c625eaef..2ef202762ff 100644
--- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py
+++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py
@@ -12,6 +12,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
DecisionsInputLimits,
+ DecisionsResponseValidationError,
ir_to_systemone_request,
ir_to_systemone_response,
parse_systemone_response,
@@ -21,6 +22,7 @@ from litellm.llms.openai.decisions.transformation import openai_request_to_ir
from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig
from litellm.types.decisions import (
MAX_DECISION_QUESTIONS,
+ DecisionsIRChoiceAnswer,
DecisionsIRRefusal,
DecisionsIRRequest,
DecisionsIRScoreAnswer,
@@ -347,6 +349,49 @@ def test_a_box_answer_without_confidence_reaches_the_caller_unchanged() -> None:
assert response.model_dump(mode="json") == payload
+@pytest.mark.parametrize(
+ ("question", "answer"),
+ [
+ (
+ {"type": "choice", "criteria": {"yes": "Yes", "no": "No"}},
+ {"type": "choice", "choice": "yes", "probabilities": {"yes": 0.9, "no": 0.1}},
+ ),
+ (
+ {"type": "score", "criteria": ["low", "high"]},
+ {"type": "score", "score": 0.5, "legend": {"0": "low", "1": "high"}, "probabilities": {"0": 0.5, "1": 0.5}},
+ ),
+ ],
+)
+@pytest.mark.parametrize("confidence", ({}, {"confidence": None}))
+@pytest.mark.parametrize(
+ ("provider", "optional"), (("nace", False), ("fireworks_ai", False), ("parallel_ai", True))
+)
+def test_only_a_provider_without_confidence_may_omit_it(
+ question: Mapping[str, object],
+ answer: Mapping[str, object],
+ confidence: Mapping[str, object],
+ provider: str,
+ optional: bool,
+) -> None:
+ config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders(provider))
+ assert config is not None
+ request: Final = systemone_request_to_ir(
+ DecisionsRequestBody.model_validate({"state": {"text": "Example"}, "questions": {"asked": question}})
+ )
+ upstream: Final = httpx.Response(
+ 200, json={"mode": "fast", "model": "decision", "answers": {"asked": {**answer, **confidence}}}
+ )
+
+ if optional:
+ parsed: Final = config.transform_decisions_response("decision", provider, upstream, request)
+ assert isinstance(parsed.answers[0], (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer))
+ assert parsed.answers[0].confidence is None
+ return
+ with pytest.raises(DecisionsResponseValidationError) as raised:
+ config.transform_decisions_response("decision", provider, upstream, request)
+ assert raised.value.status_code == 500
+
+
@pytest.mark.parametrize("provider", ("nace", "openrouter", "fireworks_ai", "typesafe"))
@pytest.mark.parametrize("legend", (["low", "high"], {"0": "low", "1": "high"}))
def test_score_legends_preserve_labels_probabilities_and_metadata(provider: str, legend: object) -> None:
diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py
index eddfe578256..186ee545323 100644
--- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py
+++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py
@@ -1,10 +1,16 @@
from collections.abc import Mapping
+from dataclasses import replace
from typing import Final
+import httpx
import pytest
from pydantic import TypeAdapter
-from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir
+from litellm.llms.base_llm.decisions.transformation import (
+ DecisionsResponseValidationError,
+ ir_to_systemone_response,
+ systemone_request_to_ir,
+)
from litellm.llms.openai.decisions.transformation import (
OpenAIDecisionsConfig,
ir_to_openai_request,
@@ -106,6 +112,35 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None:
assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
+@pytest.mark.parametrize("answer_type", ["choice", "score"])
+def test_a_choice_or_score_answer_without_confidence_stays_a_provider_error(answer_type: str) -> None:
+ answers: Final = [
+ {key: value for key, value in answer.items() if answer["type"] != answer_type or key != "confidence"}
+ for answer in _OPENAI_RESPONSE["answers"]
+ ]
+ with pytest.raises(DecisionsResponseValidationError) as raised:
+ OpenAIDecisionsConfig().transform_decisions_response(
+ "gpt-6-luna",
+ "openai",
+ httpx.Response(200, json={**_OPENAI_RESPONSE, "answers": answers}),
+ _openai_ir(_OPENAI_REQUEST),
+ )
+ assert raised.value.status_code == 500
+
+
+def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None:
+ ir: Final = _openai_ir(_OPENAI_REQUEST)
+ parsed: Final = replace(
+ OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir), extra={"provider": "upstream", "citations": []}
+ )
+
+ assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
+ assert ir_to_openai_response(parsed, ir, "requested", frozenset({"citations"})).model_dump(mode="json") == {
+ **_OPENAI_RESPONSE,
+ "citations": [],
+ }
+
+
def test_answers_openai_did_not_return_are_refusals() -> None:
ir: Final = _openai_ir(_OPENAI_REQUEST)
payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]}
diff --git a/tests/unit/llms/parallel_ai/decisions/__init__.py b/tests/unit/llms/parallel_ai/decisions/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py
new file mode 100644
index 00000000000..9cd404f61f4
--- /dev/null
+++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py
@@ -0,0 +1,423 @@
+import json
+from collections.abc import Mapping
+from dataclasses import replace
+from datetime import datetime
+from typing import Final
+
+import httpx
+import pytest
+import respx
+
+import litellm
+from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.llms.base_llm.decisions.transformation import (
+ DecisionsResponseValidationError,
+ ir_to_systemone_response,
+ systemone_request_to_ir,
+)
+from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir
+from litellm.llms.parallel_ai.decisions.transformation import ParallelAIDecisionsConfig
+from litellm.types.decisions import (
+ DecisionsIRRefusal,
+ DecisionsRequestBody,
+ OpenAIDecisionRequestBody,
+ UnsupportedDecisionsRequest,
+)
+from litellm.types.utils import LlmProviders
+from litellm.utils import ProviderConfigManager, Rules, function_setup
+
+QUESTIONS: Final = {
+ "available": {"type": "noul", "instructions": "Is this product available?"},
+ "tier": {"type": "choice", "instructions": "Which tier is offered?", "criteria": {"free": "Free", "paid": "Paid"}},
+ "fit": {"type": "score", "instructions": "How well does it fit?", "criteria": ["Poor", "Good", "Excellent"]},
+ "unknown": {"type": "noul", "instructions": "Is there enough evidence?"},
+}
+REQUEST: Final = systemone_request_to_ir(
+ DecisionsRequestBody.model_validate({"state": {"product": "Example"}, "questions": QUESTIONS})
+)
+PAYLOAD: Final = {
+ "mode": "fast",
+ "answers": {
+ "available": {"type": "noul", "noul": 0.9},
+ "tier": {"type": "choice", "choice": "paid", "probabilities": {"free": 0.2, "paid": 0.8}},
+ "fit": {
+ "type": "score",
+ "score": 1.7,
+ "legend": {"0": "Poor", "1": "Good", "2": "Excellent"},
+ "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8},
+ },
+ "unknown": {"type": "refusal"},
+ },
+ "usage": {"input_tokens": 4212, "output_tokens": 0},
+ "citations": [{"url": "https://example.com/product", "title": "Product", "excerpts": ["Available now"]}],
+ "decision_id": "decisions_example",
+}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize("surface", ["decisions", "systemone"])
+async def test_sdk_sends_parallel_wire_format_and_preserves_evidence(
+ use_async: bool, surface: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ litellm.in_memory_llm_clients_cache.flush_cache()
+ route: Final = respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD)
+ connection: Final = {"model": "parallel_ai/decision", "api_key": "test-key", "timeout": 5, "num_retries": 0}
+ fields: Final = (
+ {"state": "Example", "questions": QUESTIONS}
+ if surface == "systemone"
+ else {
+ "input": "Example",
+ "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ }
+ )
+ sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions
+ async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions
+ response: Final = await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields)
+ sent: Final = route.calls.last.request
+ body: Final = json.loads(sent.content)
+ assert body["mode"] == "fast"
+ assert "model" not in body
+ assert body["state"] == "Example"
+ assert sent.headers["x-api-key"] == "test-key"
+ assert "authorization" not in sent.headers
+ assert response.model == "decision"
+ assert response.model_dump()["citations"] == PAYLOAD["citations"]
+ assert response.model_dump()["decision_id"] == PAYLOAD["decision_id"]
+ await GLOBAL_LOGGING_WORKER.clear_queue()
+
+
+def test_provider_routes_systemone_questions_to_parallel_without_a_model_field() -> None:
+ config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders.PARALLEL_AI)
+ assert config is not None
+ assert config.transform_decisions_request("decision", REQUEST, "parallel_ai") == {
+ "mode": "fast",
+ "state": {"product": "Example"},
+ "questions": QUESTIONS,
+ }
+ assert config.get_complete_url("https://gateway.example/v1/", "decision") == "https://gateway.example/v1/decisions"
+
+
+@pytest.mark.parametrize("suffix", ["", "/", "/v1", "/v1/", "/v1/decisions", "/v1/decisions/", "/v1/search", "/v1/search/"])
+def test_base_carrying_a_parallel_route_reaches_the_decisions_endpoint(suffix: str) -> None:
+ complete_url: Final = ParallelAIDecisionsConfig().get_complete_url(f"https://api.parallel.ai{suffix}", "decision")
+ assert complete_url == "https://api.parallel.ai/v1/decisions"
+
+
+def test_auth_replaces_conflicting_headers_without_forwarding_bearer_credentials() -> None:
+ headers: Final = ParallelAIDecisionsConfig().validate_environment(
+ {"Authorization": "Bearer old", "X-API-Key": "old", "CONTENT-TYPE": "text/plain", "X-Trace": "test"},
+ "decision",
+ "test-key",
+ {},
+ )
+ assert headers == {"x-api-key": "test-key", "Content-Type": "application/json", "X-Trace": "test"}
+
+
+def test_openai_predicate_is_translated_to_a_named_noul_question() -> None:
+ request: Final = openai_request_to_ir(
+ OpenAIDecisionRequestBody.model_validate(
+ {
+ "input": "Example product",
+ "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ }
+ )
+ )
+ assert ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai") == {
+ "mode": "fast",
+ "state": "Example product",
+ "questions": {"available": {"type": "noul", "instructions": "Available?"}},
+ }
+
+
+@pytest.mark.parametrize("surface", ["systemone", "decisions"])
+def test_all_answers_usage_and_evidence_survive_response_translation(surface: str) -> None:
+ response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST)
+ assert isinstance(response.answers[3], DecisionsIRRefusal)
+ formatted: Final = (
+ ir_to_systemone_response(response, REQUEST)
+ if surface == "systemone"
+ else ir_to_openai_response(
+ response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
+ )
+ result: Final = formatted.model_dump(mode="json")
+ assert result["citations"] == PAYLOAD["citations"]
+ assert result["decision_id"] == PAYLOAD["decision_id"]
+ assert result["mode"] == "fast"
+ assert result["usage"]["input_tokens"] == 4212
+ assert result["usage"]["output_tokens"] == 0
+ if surface == "systemone":
+ assert result["answers"]["available"]["noul"] == 0.9
+ assert result["answers"]["tier"]["choice"] == "paid"
+ assert result["answers"]["fit"]["score"] == 1.7
+ assert "unknown" not in result["answers"]
+ else:
+ assert result["answers"][3]["type"] == "refusal"
+
+
+def test_rejects_more_than_one_hundred_questions() -> None:
+ request: Final = systemone_request_to_ir(
+ DecisionsRequestBody.model_validate(
+ {
+ "state": "Example",
+ "questions": {str(index): QUESTIONS["available"] for index in range(101)},
+ }
+ )
+ )
+ assert isinstance(
+ ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai"),
+ UnsupportedDecisionsRequest,
+ )
+
+
+@pytest.mark.parametrize("model", ["fast", "unknown"])
+def test_rejects_unregistered_models_instead_of_forwarding_them_as_modes(model: str) -> None:
+ assert isinstance(
+ ParallelAIDecisionsConfig().transform_decisions_request(model, REQUEST, "parallel_ai"),
+ UnsupportedDecisionsRequest,
+ )
+
+
+@pytest.mark.parametrize("confidence", [None, 0.8])
+@pytest.mark.parametrize("surface", ["systemone", "decisions"])
+def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: float | None, surface: str) -> None:
+ payload: Final = {
+ **PAYLOAD,
+ "answers": {name: {**answer, "confidence": confidence} for name, answer in PAYLOAD["answers"].items()},
+ }
+ response: Final = ParallelAIDecisionsConfig().parse_response(payload, REQUEST)
+ formatted: Final = (
+ ir_to_systemone_response(response, REQUEST)
+ if surface == "systemone"
+ else ir_to_openai_response(
+ response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
+ )
+ answers: Final = formatted.model_dump(mode="json")["answers"]
+ values: Final = answers.values() if isinstance(answers, dict) else answers
+ assert [answer["confidence"] for answer in values if answer["type"] in {"choice", "score"}] == [confidence] * 2
+
+
+def test_provider_extras_cannot_replace_normalized_response_fields() -> None:
+ response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST)
+ with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}})
+ formatted: Final = ir_to_openai_response(
+ with_collision, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
+ assert formatted.model == "decision"
+ assert len(formatted.answers) == 4
+ assert formatted.usage.input_tokens == 4212
+ assert formatted.model_dump()["citations"] == PAYLOAD["citations"]
+
+
+def test_undocumented_parallel_fields_stay_off_the_openai_response() -> None:
+ response: Final = ParallelAIDecisionsConfig().parse_response({**PAYLOAD, "internal_trace_id": "trace"}, REQUEST)
+ formatted: Final = ir_to_openai_response(
+ response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
+ assert "internal_trace_id" not in formatted.model_dump()
+
+
+@pytest.mark.parametrize("tokens", [0, 100, 32000])
+@pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"])
+def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None:
+ rate: Final = litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"]
+ cost: Final = litellm.cost_per_token(
+ model="parallel_ai/decision",
+ prompt_tokens=tokens,
+ completion_tokens=0,
+ call_type=call_type,
+ )
+ assert cost == (rate, 0.0)
+
+
+@pytest.mark.asyncio
+async def test_deployment_query_price_replaces_the_catalog_price(
+ respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ litellm.in_memory_llm_clients_cache.flush_cache()
+ respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD)
+ price: Final = 25 * litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"]
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "webjev",
+ "litellm_params": {
+ "model": "parallel_ai/decision",
+ "api_key": "test-key",
+ "input_cost_per_query": price,
+ },
+ }
+ ]
+ )
+ fields: Final = {
+ "model": "webjev",
+ "input": "Example",
+ "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ "litellm_call_id": "query-price-call",
+ }
+ logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields)
+ response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj)
+ assert response._hidden_params["response_cost"] == pytest.approx(price)
+ await GLOBAL_LOGGING_WORKER.clear_queue()
+
+
+@pytest.mark.asyncio
+async def test_query_price_on_another_provider_leaves_its_token_billing_alone(
+ respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ litellm.in_memory_llm_clients_cache.flush_cache()
+ input_rate: Final = 1e-06
+ tokens: Final = 1000
+ model: Final = "fireworks_ai/accounts/example/models/token-billed"
+ monkeypatch.setitem(
+ litellm.model_cost,
+ model,
+ {"input_cost_per_token": input_rate, "output_cost_per_token": 0.0}
+ | {"litellm_provider": "fireworks_ai", "mode": "evaluation"},
+ )
+ respx_mock.post("https://api.fireworks.ai/inference/v1/systemone").respond(
+ json={
+ "model": "token-billed",
+ "answers": {"available": {"type": "noul", "noul": 0.9}},
+ "usage": {"input_tokens": tokens, "output_tokens": 0},
+ }
+ )
+ router: Final = litellm.Router(
+ model_list=[
+ {
+ "model_name": "token-billed",
+ "litellm_params": {"model": model, "api_key": "test-key", "input_cost_per_query": 0.025},
+ }
+ ]
+ )
+ fields: Final = {
+ "model": "token-billed",
+ "input": "Example",
+ "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ "litellm_call_id": "token-billed-call",
+ }
+ logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields)
+ response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj)
+ assert response._hidden_params["response_cost"] == pytest.approx(tokens * input_rate)
+ await GLOBAL_LOGGING_WORKER.clear_queue()
+
+
+@pytest.mark.parametrize(
+ "payload", [{"answers": {}}, {"mode": "fast", "answers": {"available": {"type": "noul", "noul": "invalid"}}}]
+)
+def test_malformed_provider_responses_raise_a_provider_error(payload: Mapping[str, object]) -> None:
+ with pytest.raises(DecisionsResponseValidationError):
+ ParallelAIDecisionsConfig().transform_decisions_response(
+ "decision", "parallel_ai", httpx.Response(200, json=payload), REQUEST
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize("surface", ["decisions", "systemone"])
+@pytest.mark.parametrize("key_env", ["PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"])
+async def test_untrusted_base_cannot_receive_environment_key(
+ use_async: bool, surface: str, key_env: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False)
+ monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False)
+ monkeypatch.delenv("PARALLEL_API_KEY", raising=False)
+ monkeypatch.setenv(key_env, "server-secret")
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ route: Final = respx_mock.post("https://attacker.example/v1/decisions").respond(json=PAYLOAD)
+ connection: Final = {"model": "parallel_ai/decision", "api_base": "https://attacker.example", "num_retries": 0}
+ fields: Final = (
+ {"state": "Example", "questions": QUESTIONS}
+ if surface == "systemone"
+ else {
+ "input": "Example",
+ "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ }
+ )
+ sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions
+ async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions
+
+ async def call() -> object:
+ return await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields)
+
+ with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
+ await call()
+ assert not route.called
+ await GLOBAL_LOGGING_WORKER.clear_queue()
+
+
+@pytest.mark.parametrize(
+ "api_base",
+ [
+ "https://api.parallel.ai.attacker.example",
+ "https://api.parallel.ai@attacker.example",
+ "http://api.parallel.ai",
+ "https://api.parallel.ai:8443",
+ "https://attacker.example/api.parallel.ai",
+ "https://api.parallel.ai/other-tenant",
+ "https://api.parallel.ai/other-tenant/v1/search",
+ "https://attacker.example/v1/search",
+ "https://attacker.example/v1/decisions",
+ ],
+)
+def test_environment_key_rejects_untrusted_destination_variants(api_base: str, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False)
+ monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")
+ with pytest.raises(ValueError, match="explicit api_key"):
+ ParallelAIDecisionsConfig().resolve_connection(api_base, None)
+
+
+@pytest.mark.parametrize(
+ "api_base",
+ [
+ None,
+ "https://api.parallel.ai",
+ "https://API.PARALLEL.AI:443/v1/",
+ "https://api.parallel.ai/v1/decisions",
+ "https://api.parallel.ai/v1/search",
+ ],
+)
+def test_default_destination_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False)
+ monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")
+ resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None)
+ assert resolved_base == (api_base or "https://api.parallel.ai")
+ assert key == "server-secret"
+
+
+@pytest.mark.parametrize(
+ "api_base", [None, "https://gateway.example/parallel/v1/", "https://gateway.example/parallel/v1/search"]
+)
+def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://gateway.example/parallel")
+ monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")
+ resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None)
+ assert resolved_base == (api_base or "https://gateway.example/parallel")
+ assert key == "server-secret"
+
+
+@pytest.mark.parametrize("api_key", [None, "caller-key"])
+@pytest.mark.parametrize("api_base", ["https://api.parallel.ai\t", "https://api.parallel.ai\r\nHost: attacker.example"])
+def test_malformed_destination_is_a_bad_request(api_base: str, api_key: str | None) -> None:
+ with pytest.raises(litellm.BadRequestError):
+ litellm.decisions(
+ model="parallel_ai/decision",
+ api_base=api_base,
+ api_key=api_key,
+ input="Example",
+ questions=[{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ num_retries=0,
+ )
+
+
+def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")
+ assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == (
+ "https://gateway.example",
+ "caller-key",
+ )
diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py
index 72755633376..d399adaea7d 100644
--- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py
+++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py
@@ -536,6 +536,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
(
("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE),
("Nace", "Nace", LlmProviders.NACE),
+ ("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI),
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
),
)
diff --git a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py
index c2931470482..ce1dfce1646 100644
--- a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py
+++ b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py
@@ -32,6 +32,29 @@ from litellm.router import Router
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
+@pytest.mark.parametrize(
+ ("model", "expected_free"),
+ [("parallel_ai/decision", False), ("fireworks_ai/accounts/fireworks/routers/rune", True)],
+)
+def test_query_price_keeps_budgets_only_where_it_is_billed(model: str, expected_free: bool) -> None:
+ router = Router(
+ model_list=[
+ {
+ "model_name": "query-priced",
+ "litellm_params": {
+ "model": model,
+ "api_key": "test-key",
+ "input_cost_per_token": 0.0,
+ "output_cost_per_token": 0.0,
+ "input_cost_per_query": 0.025,
+ },
+ }
+ ]
+ )
+
+ assert is_model_cost_zero("query-priced", router) is expected_free
+
+
@pytest.fixture
def mock_router_with_zero_cost_model():
"""Create a mock router with a zero-cost model."""
diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py
index ef7b9e485d4..d0e2b381253 100644
--- a/tests/unit/router_strategy/test_complexity_router.py
+++ b/tests/unit/router_strategy/test_complexity_router.py
@@ -359,6 +359,52 @@ class TestComplexityRouterInit:
"tier-probability:MEDIUM=0.900000",
)
+ @pytest.mark.asyncio
+ async def test_jev_choice_without_confidence_routes_and_reports_no_confidence(self, mock_router_instance):
+ client = _StaticJevClient(
+ JevSystemOneResponse(
+ answers={
+ "tier": JevChoiceAnswer(
+ type="choice",
+ choice="MEDIUM",
+ probabilities={"SIMPLE": 0.1, "MEDIUM": 0.9},
+ )
+ },
+ )
+ )
+ router = ComplexityRouter(
+ "test-router",
+ mock_router_instance,
+ {
+ "classifier_type": "jev",
+ "jev_classifier_config": {"api_key": "test", "timeout_ms": 100},
+ "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
+ },
+ derive_savings_baseline=False,
+ jev_client=client,
+ )
+
+ outcome = await router.aclassify("Explain this")
+ result = await router.async_pre_routing_hook(
+ model="test-router",
+ request_kwargs={},
+ messages=[{"role": "user", "content": "Explain this"}],
+ )
+
+ assert outcome.tier == ComplexityTier.MEDIUM
+ assert outcome.cause == "jev_classifier"
+ assert outcome.jev_verdict is not None
+ assert outcome.jev_verdict.confidence is None
+ assert outcome.signals == (
+ "jev-classifier:MEDIUM",
+ "tier-probability:SIMPLE=0.100000",
+ "tier-probability:MEDIUM=0.900000",
+ )
+ assert result is not None
+ assert result.routing_decision is not None
+ assert result.routing_decision["classifier_probabilities"] == {"SIMPLE": 0.1, "MEDIUM": 0.9}
+ assert "classifier_confidence" not in result.routing_decision
+
@pytest.mark.asyncio
async def test_jev_pre_routing_hook_exposes_routing_decision_provenance(
self, mock_router_instance, monkeypatch: pytest.MonkeyPatch
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx
index f6c5794fa5c..9c78eff1a2b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx
@@ -109,6 +109,20 @@ describe("ResponseView integration", () => {
expect(screen.getByText("378 in / 63 out")).toBeInTheDocument();
});
+ it("leaves out the confidence badge when the provider reports no confidence", () => {
+ const withoutConfidence: SystemOneResponse = {
+ ...sampleResponse,
+ answers: {
+ dept: { type: "choice", choice: "technical", confidence: null, probabilities: { technical: 1 } },
+ frustration: { type: "score", score: 2, confidence: null, probabilities: { "2": 1 } },
+ },
+ };
+
+ render();
+
+ expect(screen.queryByText(/% confidence/)).not.toBeInTheDocument();
+ });
+
it("selects the score level nearest a fractional calibrated score", () => {
const fractional: SystemOneResponse = {
answers: {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx
index 572bf765ac1..89d998b2632 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx
@@ -71,7 +71,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
Selected choice
{answer.choice}
- {answer.confidence !== undefined && (
+ {typeof answer.confidence === "number" && (
{Math.round(answer.confidence * 100)}% confidence
)}
@@ -90,7 +90,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
Score
{SCORE_FORMAT.format(answer.score)}
- {answer.confidence !== undefined && (
+ {typeof answer.confidence === "number" && (
{Math.round(answer.confidence * 100)}% confidence
)}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts
index 0e06d733ea3..1b906a8d975 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts
@@ -62,14 +62,14 @@ const choiceAnswerFields = {
...answerFields,
type: z.literal("choice"),
choice: choiceValue,
- confidence: probability,
+ confidence: probability.nullish(),
probabilities: z.array(z.looseObject({ value: choiceValue, probability })),
};
const scoreAnswerFields = {
...answerFields,
type: z.literal("score"),
score: z.number().finite(),
- confidence: probability,
+ confidence: probability.nullish(),
probabilities: z.array(z.looseObject({ value: z.number().int(), label: z.string(), probability })),
};
const usageFields = {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts
index 7c923fe8e8b..108da052893 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts
@@ -125,7 +125,7 @@ const noulAnswerSchema = z.looseObject({ type: z.literal("noul"), noul: probabil
const choiceAnswerShape = {
type: z.literal("choice"),
choice: z.string(),
- confidence: probability.optional(),
+ confidence: probability.nullish(),
probabilities,
};
const choiceAnswerSchema = z.looseObject(choiceAnswerShape);
@@ -133,7 +133,7 @@ const choiceAnswerSchema = z.looseObject(choiceAnswerShape);
const scoreAnswerShape = {
type: z.literal("score"),
score: z.number().finite(),
- confidence: probability.optional(),
+ confidence: probability.nullish(),
legend: z.record(z.string(), decisionsJson).optional(),
probabilities,
};
diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx
index 4c483434831..c88e44f21d4 100644
--- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx
+++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx
@@ -73,6 +73,12 @@ describe("provider_info_helpers", () => {
});
describe("getProviderLogoAndName", () => {
+ it("shows the Parallel AI logo for the parallel_ai slug", () => {
+ const { logo, displayName } = getProviderLogoAndName("parallel_ai");
+ expect(displayName).toBe(Providers.PARALLEL_AI);
+ expect(logo).toContain("parallel_ai");
+ });
+
it("should return empty logo and dash display name when providerValue is empty", () => {
const result = getProviderLogoAndName("");
expect(result).toEqual({ logo: "", displayName: "-" });
@@ -577,6 +583,16 @@ describe("provider_info_helpers", () => {
expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]);
});
+ it("lists Parallel decision models with their provider and model placeholder", () => {
+ const modelMap = {
+ "parallel_ai/decision": { litellm_provider: "parallel_ai", mode: "evaluation" },
+ };
+
+ expect(getProviderModels("PARALLEL_AI", modelMap)).toEqual(["parallel_ai/decision"]);
+ expect(getProviderLogoAndName("parallel_ai").displayName).toBe("Parallel AI");
+ expect(getPlaceholder("PARALLEL_AI")).toBe("parallel_ai/decision");
+ });
+
it("lists Nace decision models and resolves the provider name and model placeholder", () => {
const modelMap = {
"nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" },
diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx
index a8879eb570b..1680258e10b 100644
--- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx
+++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx
@@ -50,6 +50,7 @@ import ollamaLogo from "../../public/assets/logos/ollama.svg";
import openaiSmallLogo from "../../public/assets/logos/openai_small.svg";
import openrouterLogo from "../../public/assets/logos/openrouter.svg";
import oracleLogo from "../../public/assets/logos/oracle.svg";
+import parallelAiLogo from "../../public/assets/logos/parallel_ai.png";
import perplexityAiLogo from "../../public/assets/logos/perplexity-ai.svg";
import qwenLogo from "../../public/assets/logos/qwen.png";
import recraftLogo from "../../public/assets/logos/recraft.svg";
@@ -191,6 +192,7 @@ export enum Providers {
Triton = "Triton",
TypeSafe = "TypeSafe",
Nace = "Nace",
+ PARALLEL_AI = "Parallel AI",
V0 = "V0",
VERCEL_AI_GATEWAY = "Vercel Ai Gateway",
Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)",
@@ -325,6 +327,7 @@ export const provider_map: Record = {
Triton: "triton",
TypeSafe: "typesafe",
Nace: "nace",
+ PARALLEL_AI: "parallel_ai",
V0: "v0",
VERCEL_AI_GATEWAY: "vercel_ai_gateway",
Vertex_AI: "vertex_ai",
@@ -419,6 +422,7 @@ export const providerLogoMap: Partial> = {
[Providers.OpenAI_Compatible]: openaiSmallLogo.src,
[Providers.Openrouter]: openrouterLogo.src,
[Providers.Oracle]: oracleLogo.src,
+ [Providers.PARALLEL_AI]: parallelAiLogo.src,
[Providers.Perplexity]: perplexityAiLogo.src,
[Providers.Qwen_AI_Platform]: qwenLogo.src,
[Providers.QwenCloud]: qwenLogo.src,
@@ -514,6 +518,7 @@ const providerPlaceholderMap: Partial> = {
[Providers.Tencent]: "tencent/deepseek-v4-pro",
[Providers.TypeSafe]: "typesafe/jev-latest",
[Providers.Nace]: "nace/drex-latest",
+ [Providers.PARALLEL_AI]: "parallel_ai/decision",
[Providers.StreamLake]: "streamlake/",
[Providers.Vertex_AI]: "gemini-pro",
[Providers.VolcEngine]: "volcengine/",
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 8278fd9fff4..79680f096e3 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -29404,7 +29404,7 @@ export interface components {
/** Choice */
choice: string;
/** Confidence */
- confidence: number;
+ confidence?: number | null;
/** Probabilities */
probabilities: {
[key: string]: number;
@@ -40226,7 +40226,7 @@ export interface components {
/** Choice */
choice: string | boolean;
/** Confidence */
- confidence: number;
+ confidence?: number | null;
/** Name */
name: string | null;
/** Probabilities */
@@ -40322,7 +40322,7 @@ export interface components {
/** OpenAIScoreAnswer */
OpenAIScoreAnswer: {
/** Confidence */
- confidence: number;
+ confidence?: number | null;
/** Name */
name: string | null;
/** Probabilities */
@@ -45564,7 +45564,7 @@ export interface components {
/** ScoreAnswer */
ScoreAnswer: {
/** Confidence */
- confidence: number;
+ confidence?: number | null;
/** Legend */
legend: {
[key: string]: string | {
From 5e08cb0e632f0fd36ab8be0c7d4584fadbdffd38 Mon Sep 17 00:00:00 2001
From: Nate Armstrong
Date: Mon, 2 Nov 2026 21:02:52 -0800
Subject: [PATCH 02/25] test(messages): add rust vs python messages parity
characterization suite (#49610)
Runs every case through the Python /v1/messages path and the Rust path against one recording upstream and compares dispatch, upstream request, outcome, hidden params, standard logging fields and callback kwargs. Known Rust differences are strict xfails tagged with their inventory ID.
---
.../strategies/e2e_parity/__init__.py | 2 +-
.../e2e_parity/sdk/messages/conftest.py | 34 +
.../sdk/messages/test_native_parity.py | 1200 +++++++++++++++++
.../support/recording_server.py | 5 +-
4 files changed, 1239 insertions(+), 2 deletions(-)
create mode 100644 tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py
create mode 100644 tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py
diff --git a/tests/rust-python-harness/strategies/e2e_parity/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/__init__.py
index 298c75f2af1..a2ea98f19ab 100644
--- a/tests/rust-python-harness/strategies/e2e_parity/__init__.py
+++ b/tests/rust-python-harness/strategies/e2e_parity/__init__.py
@@ -20,7 +20,7 @@ CASES: Final[tuple[CaseDefinition, ...]] = (
CaseDefinition(
"messages",
NotImplementedCaseSpec(
- reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
+ reason="Parity runs as the pytest suite in sdk/messages/test_native_parity.py, not as a runner case."
),
surface="sdk",
),
diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py
new file mode 100644
index 00000000000..4bb20713c59
--- /dev/null
+++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py
@@ -0,0 +1,34 @@
+from collections.abc import AsyncIterator, Generator
+from importlib import import_module
+from pathlib import Path
+from typing import Final
+
+import pytest
+import pytest_asyncio
+
+from tests.test_litellm_rust.support.isolation import isolated_test_state
+from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service
+
+PARITY_ROOT: Final = Path(__file__).parent
+
+
+@pytest_asyncio.fixture(autouse=True, loop_scope="function")
+async def isolate_test_state() -> AsyncIterator[None]:
+ async with isolated_test_state():
+ yield
+
+
+@pytest.fixture
+def recording_server() -> Generator[RecordingServer]:
+ with recording_service() as server:
+ yield server
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+ try:
+ import_module("litellm.rust_bridge._native")
+ except ImportError:
+ skip: Final = pytest.mark.skip(reason="requires a compiled litellm.rust_bridge._native extension")
+ for item in items:
+ if PARITY_ROOT in item.path.parents:
+ item.add_marker(skip)
diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py
new file mode 100644
index 00000000000..cdfba773cf6
--- /dev/null
+++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py
@@ -0,0 +1,1200 @@
+"""Native Anthropic Messages calls must be indistinguishable from the Python implementation.
+
+Each case sends one `/v1/messages` SDK call through the Python handler and through the native route against the same
+scripted upstream, then compares one aspect of what happened:
+
+- dispatch: the native leg was served natively instead of falling back to Python
+- request: every request that reached the upstream (method, path, body, headers)
+- outcome: what the caller got back (response body, stream bytes, or the raised exception)
+- hidden_params: the `_hidden_params` keys the proxy builds its response headers from
+- logged: which terminal callbacks fired and the stable standard logging payload fields
+- logged_metadata, logged_model_info, logged_response_headers, logged_api_key, logged_call: single callback kwargs
+ fields that proxy integrations read
+
+Both legs run once per case and every aspect of that case reads the same pair
+
+The native leg prepends an unconstrained Rust rule, so feature detectors that keep a call on Python today do not
+apply: a case for a detector-covered feature shows what Rust still lacks before that detector can go. Every known
+difference is a strict xfail naming the gap, with an id from the Rust vs Python Messages inventory (A routing,
+R request, S response, C callbacks, F covered by a fallback detector today)
+"""
+
+import asyncio
+import json
+import os
+import re
+from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass, field, replace
+from types import MappingProxyType
+from typing import Final, cast
+
+import pytest
+from pydantic import JsonValue, TypeAdapter
+
+import litellm
+from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
+from litellm.messages import dispatch
+from litellm.rust_bridge import catalog
+from litellm.rust_bridge.catalog import Route, RouteRule
+from litellm.rust_bridge.configuration import Rollout
+from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging
+from tests.test_litellm_rust.support.isolation import isolated_callback_registries, rebound
+from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
+
+pytestmark = pytest.mark.requires_rust_extension
+
+SONNET: Final = "anthropic/claude-sonnet-4-5"
+OPUS_ADAPTIVE: Final = "anthropic/claude-opus-4-7"
+OPUS_FAST: Final = "anthropic/claude-opus-5-5"
+API_KEY: Final = "sk-ant-parity-test"
+USER: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": "What is the capital of France?"})
+
+_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_LOGGED: Final = TypeAdapter(dict[str, object])
+_TRANSPORT_HEADERS: Final = frozenset(
+ {"accept", "accept-encoding", "connection", "content-length", "host", "user-agent"}
+)
+_COMPARED_LOG_FIELDS: Final = (
+ "call_type",
+ "custom_llm_provider",
+ "model",
+ "model_group",
+ "model_id",
+ "api_base",
+ "stream",
+ "cache_hit",
+ "prompt_tokens",
+ "completion_tokens",
+ "total_tokens",
+ "response_cost",
+ "status",
+)
+_TERMINAL_EVENTS: Final = (
+ "log_success_event",
+ "async_log_success_event",
+ "log_failure_event",
+ "async_log_failure_event",
+)
+_VOLATILE: Final = (
+ (re.compile(r"time taken=[0-9.]+"), "time taken="),
+ (re.compile(r"Timeout passed=[0-9.]+"), "Timeout passed="),
+ (re.compile(r"127\.0\.0\.1:[0-9]+"), "127.0.0.1:"),
+)
+
+TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType(
+ {
+ "id": "msg_01ParityText",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-5-20250929",
+ "content": [{"type": "text", "text": "Paris"}],
+ "stop_reason": "end_turn",
+ "stop_sequence": None,
+ "usage": {"input_tokens": 14, "output_tokens": 4},
+ }
+)
+
+TOOL_USE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType(
+ {
+ **TEXT_RESPONSE,
+ "id": "msg_01ParityTool",
+ "content": [
+ {"type": "text", "text": "Checking."},
+ {"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {"city": "Paris"}},
+ ],
+ "stop_reason": "tool_use",
+ }
+)
+
+THINKING_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType(
+ {
+ **TEXT_RESPONSE,
+ "id": "msg_01ParityThinking",
+ "content": [
+ {"type": "thinking", "thinking": "The capital of France is Paris.", "signature": "EqQBCkYIARgCKkB"},
+ {"type": "text", "text": "Paris"},
+ ],
+ }
+)
+
+CACHED_USAGE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType(
+ {
+ **TEXT_RESPONSE,
+ "usage": {
+ "input_tokens": 14,
+ "output_tokens": 4,
+ "cache_creation_input_tokens": 2048,
+ "cache_read_input_tokens": 1024,
+ "service_tier": "standard",
+ },
+ }
+)
+
+WEATHER_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType(
+ {
+ "name": "get_weather",
+ "description": "Current weather for a city",
+ "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
+ }
+)
+
+TOOL_TURNS: Final[tuple[JsonValue, ...]] = (
+ {"role": "user", "content": "Weather in Paris?"},
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {"city": "Paris"}}],
+ },
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01Parity", "content": "18C, sunny"}]},
+)
+
+THINKING_REPLAY: Final[tuple[JsonValue, ...]] = (
+ {"role": "user", "content": "What is the capital of France?"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "Paris.", "signature": "EqQBCkYIARgCKkB"},
+ {"type": "text", "text": "Paris"},
+ ],
+ },
+ {"role": "user", "content": "And Germany?"},
+)
+
+NO_INPUT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
+PNG_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+
+
+def _events(*events: tuple[str, JsonValue]) -> tuple[tuple[str, object], ...]:
+ return tuple((name, data) for name, data in events)
+
+
+def _message_start(message_id: str) -> tuple[str, JsonValue]:
+ return (
+ "message_start",
+ {
+ "type": "message_start",
+ "message": {
+ **TEXT_RESPONSE,
+ "id": message_id,
+ "content": [],
+ "stop_reason": None,
+ "usage": {"input_tokens": 14, "output_tokens": 1},
+ },
+ },
+ )
+
+
+def _text_block(index: int, *pieces: str) -> tuple[tuple[str, JsonValue], ...]:
+ return (
+ (
+ "content_block_start",
+ {"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}},
+ ),
+ *(
+ (
+ "content_block_delta",
+ {"type": "content_block_delta", "index": index, "delta": {"type": "text_delta", "text": piece}},
+ )
+ for piece in pieces
+ ),
+ ("content_block_stop", {"type": "content_block_stop", "index": index}),
+ )
+
+
+def _message_end(stop_reason: str, output_tokens: int) -> tuple[tuple[str, JsonValue], ...]:
+ return (
+ (
+ "message_delta",
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": stop_reason, "stop_sequence": None},
+ "usage": {"output_tokens": output_tokens},
+ },
+ ),
+ ("message_stop", {"type": "message_stop"}),
+ )
+
+
+TEXT_STREAM: Final = _events(
+ _message_start("msg_01ParityStream"), *_text_block(0, "Pa", "ris"), *_message_end("end_turn", 4)
+)
+
+TOOL_STREAM: Final = _events(
+ _message_start("msg_01ParityToolStream"),
+ *_text_block(0, "Checking."),
+ (
+ "content_block_start",
+ {
+ "type": "content_block_start",
+ "index": 1,
+ "content_block": {"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {}},
+ },
+ ),
+ (
+ "content_block_delta",
+ {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": '}},
+ ),
+ (
+ "content_block_delta",
+ {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '"Paris"}'}},
+ ),
+ ("content_block_stop", {"type": "content_block_stop", "index": 1}),
+ *_message_end("tool_use", 22),
+)
+
+THINKING_STREAM: Final = _events(
+ _message_start("msg_01ParityThinkingStream"),
+ (
+ "content_block_start",
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {"type": "thinking", "thinking": "", "signature": ""},
+ },
+ ),
+ (
+ "content_block_delta",
+ {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "Paris."}},
+ ),
+ (
+ "content_block_delta",
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "signature_delta", "signature": "EqQBCkYIARgCKkB"},
+ },
+ ),
+ ("content_block_stop", {"type": "content_block_stop", "index": 0}),
+ *_text_block(1, "Paris"),
+ *_message_end("end_turn", 12),
+)
+
+ERROR_MID_STREAM: Final = _events(
+ _message_start("msg_01ParityOverloaded"),
+ *_text_block(0, "Pa"),
+ ("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}),
+)
+
+SLOW_STREAM: Final = _events(
+ _message_start("msg_01ParitySlow"),
+ *_text_block(0, "a", "b", "c", "d", "e", "f"),
+ *_message_end("end_turn", 6),
+)
+
+
+def _upstream_error(kind: str, message: str) -> Mapping[str, JsonValue]:
+ return MappingProxyType({"type": "error", "error": {"type": kind, "message": message}})
+
+
+def _ok(body: Mapping[str, JsonValue] = TEXT_RESPONSE, **headers: str) -> ResponseSpec:
+ return ResponseSpec(body=dict(body), headers=dict(headers))
+
+
+def _stream(events: tuple[tuple[str, object], ...], chunk_delay: float = 0) -> ResponseSpec:
+ return ResponseSpec(body={}, events=events, chunk_delay=chunk_delay)
+
+
+def _failure(status: int, kind: str, message: str, **headers: str) -> ResponseSpec:
+ return ResponseSpec(body=dict(_upstream_error(kind, message)), status=status, headers=dict(headers))
+
+
+CALLBACK_ASPECTS: Final = (
+ "logged_metadata",
+ "logged_model_info",
+ "logged_response_headers",
+ "logged_api_key",
+ "logged_call",
+)
+ASPECTS: Final = ("dispatch", "request", "outcome", "hidden_params", "logged", *CALLBACK_ASPECTS)
+
+
+@dataclass(frozen=True, slots=True)
+class Case:
+ name: str
+ kwargs: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
+ upstream: tuple[ResponseSpec, ...] = (ResponseSpec(body=dict(TEXT_RESPONSE)),)
+ model: str = SONNET
+ sync: bool = False
+ env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
+ settings: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
+ private_keys: bool = False
+ gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({}))
+
+ def params(self) -> Iterator[object]:
+ for aspect in ASPECTS:
+ gap = {**KNOWN_GAPS.get(self.name, {}), **self.gaps}.get(aspect)
+ marks = () if not gap else (pytest.mark.xfail(reason=gap, strict=True),)
+ yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks)
+
+
+def _gaps(**gaps: str | None) -> Mapping[str, str | None]:
+ return MappingProxyType(gaps)
+
+
+def _kw(**kwargs: object) -> Mapping[str, object]:
+ return MappingProxyType(kwargs)
+
+
+A2_BARE_MODEL: Final = "A2: native resolves the provider from the model prefix only, so a bare Claude model name fails"
+A10_NO_MAX_TOKENS: Final = "A10: native admission declines a call without an int max_tokens and Python serves it"
+R3_LEGACY_THINKING: Final = "R3: native maps a float legacy thinking budget to effort low instead of Python's tier"
+R4_BINDING_BETA: Final = "R4: native omits the thinking-binding-controls beta for block_binding"
+R5_DEPLOYMENT_EFFORT: Final = "R5: native ignores the deployment's reasoning_effort_levels from model_info"
+R7_DROP_ORDER: Final = "R7: native applies additional_drop_params before Python's unsupported-param check"
+R8_VERSION_HEADER: Final = "R8: Python sends anthropic-version twice when the caller capitalises it; native once"
+R9_QUERY_PATH: Final = "R9: native appends /v1/messages again to an api_base that ends in /v1/messages?query"
+R11_PARAM_ERROR: Final = "R11: native raises BadRequestError where Python raises UnsupportedParamsError"
+R12_METADATA_ERROR: Final = (
+ "R12: Python raises a pydantic ValidationError for a non-string metadata.user_id; native a BadRequestError"
+)
+S1_S2_ERROR_HEADERS: Final = (
+ "S1/S2: native drops upstream error headers and wraps the error body in a (status, body) tuple"
+)
+S2_ERROR_MESSAGE: Final = (
+ "S2: native wraps the upstream error body in a (status, body) tuple and strips the provider from model"
+)
+S2_S12_ERROR_STATUS: Final = "S2/S12: native keeps the upstream status where Python maps it (413 to 400, 529 to 500)"
+S3_TIMEOUT: Final = "S3: native raises APIConnectionError (500) for a timeout where Python raises Timeout (408)"
+S5_STREAM_DEADLINE: Final = "S5: native applies the timeout to the whole stream instead of to each read"
+S6_PRIVATE_KEYS: Final = "S6: native writes _hidden_params into the response dict"
+S8_TYPED_RESPONSE: Final = "S8: native re-serializes the response and drops null usage and container"
+S9_SYNC: Final = "S9: Python's sync handler raises ValueError for native providers; native serves the call"
+S13_INVALID_JSON: Final = "S13: native reports an unparseable success body as an internal error, not AnthropicException"
+S14_MISSING_KEY: Final = (
+ "S14: native raises APIConnectionError for a missing API key where Python raises AuthenticationError"
+)
+C4_METADATA: Final = "C4: native callbacks get the raw Anthropic metadata, not Python's merged litellm metadata"
+C6_RESPONSE_HEADERS: Final = "C6: native success logs carry no response_headers"
+C7_API_KEY: Final = "C7: native callbacks see the caller's api_key where Python logs an empty string"
+C8_STREAM_HIDDEN: Final = "C8: native streams lack the call metadata @client stamps into _hidden_params"
+F_MOCK: Final = "F: mock_response is Python-only (detector mock_response)"
+F_THINKING_RETRY: Final = (
+ "F: retry without thinking after a rejected signature is Python-only (detector thinking_signature_retry)"
+)
+F_CACHE_INJECTION: Final = "F: prompt cache_control injection is Python-only (detector cache_control_injection)"
+F_TOOL_CUSTOM: Final = "F: hoisting a tool's custom field is Python-only (detector tool_custom_field)"
+F_STREAM_COST: Final = "F: cost injection into streamed usage is Python-only (detector stream_cost_injection)"
+
+KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType(
+ {
+ "plain_text": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "system_string": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "system_blocks_with_cache_control": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "multi_turn_tool_result": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "tools_with_tool_choice": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "tool_choice_none": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "tool_input_examples": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "sampling_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "stop_sequences": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "metadata_user_id": _gaps(
+ logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "metadata_extra_keys_are_filtered": _gaps(
+ logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "metadata_user_id_not_a_string": _gaps(outcome=R12_METADATA_ERROR, logged=R12_METADATA_ERROR),
+ "service_tier": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "thinking_enabled": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "thinking_disabled": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "legacy_thinking_on_adaptive_model_keeps_display": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "legacy_thinking_float_budget_on_adaptive_model": _gaps(
+ request=R3_LEGACY_THINKING, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "adaptive_thinking_with_block_binding": _gaps(
+ request=R4_BINDING_BETA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "reasoning_effort": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "reasoning_effort_on_adaptive_model": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "output_config_effort": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "output_format_json_schema": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "speed_on_model_without_speed": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR),
+ "speed_on_model_without_speed_drop_params": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "speed_on_fast_model": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "temperature_on_adaptive_model": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR),
+ "top_k_on_adaptive_model": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR),
+ "top_k_on_adaptive_model_drop_params": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "additional_drop_params_hides_rejected_temperature": _gaps(
+ request=R7_DROP_ORDER,
+ outcome=R7_DROP_ORDER,
+ hidden_params=R7_DROP_ORDER,
+ logged=R7_DROP_ORDER,
+ logged_metadata=R7_DROP_ORDER,
+ logged_api_key=R7_DROP_ORDER,
+ logged_call=R7_DROP_ORDER,
+ ),
+ "additional_drop_params_nested_path": _gaps(
+ logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "context_management_edits": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "container_skills": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "remote_mcp_servers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "inference_geo": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "top_level_cache_control": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "null_safeguards_are_stripped": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "empty_text_blocks_are_stripped": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "tool_use_ids_are_sanitized": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "provider_specific_fields_are_stripped": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "image_block": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "document_block": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "server_web_search_tool": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "extra_headers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "anthropic_beta_from_headers_and_extra_headers": _gaps(
+ logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "caller_anthropic_version_capitalised": _gaps(
+ request=R8_VERSION_HEADER, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "provider_specific_header": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "explicit_custom_llm_provider": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "bare_model_name": _gaps(**dict.fromkeys(ASPECTS[1:], A2_BARE_MODEL) | {"logged_model_info": None}),
+ "api_base_with_messages_path": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "api_base_with_messages_path_and_query": _gaps(
+ request=R9_QUERY_PATH,
+ logged=R9_QUERY_PATH,
+ logged_response_headers=C6_RESPONSE_HEADERS,
+ logged_api_key=C7_API_KEY,
+ ),
+ "api_key_from_environment": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "auth_token_from_environment": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "reasoning_auto_summary": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "global_drop_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "request_timeout_setting": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "litellm_metadata_and_metadata": _gaps(
+ logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "model_info_with_id": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "deployment_effort_levels": _gaps(
+ request=R5_DEPLOYMENT_EFFORT, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "response_with_tool_use": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_with_thinking": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_with_cache_usage": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_with_null_usage_and_container": _gaps(outcome=S8_TYPED_RESPONSE),
+ "response_without_stop_sequence": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_with_unknown_field": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_key_order": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_upstream_headers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "response_private_keys": _gaps(
+ outcome=S6_PRIVATE_KEYS, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "upstream_400": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_401": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_403": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_404": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_413": _gaps(outcome=S2_S12_ERROR_STATUS, logged_api_key=C7_API_KEY),
+ "upstream_429_with_retry_after": _gaps(outcome=S1_S2_ERROR_HEADERS, logged_api_key=C7_API_KEY),
+ "upstream_500": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_529_overloaded": _gaps(outcome=S2_S12_ERROR_STATUS, logged_api_key=C7_API_KEY),
+ "upstream_400_context_window": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_non_json_error": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "upstream_invalid_json_success": _gaps(
+ outcome=S13_INVALID_JSON, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "upstream_timeout": _gaps(outcome=S3_TIMEOUT, logged=S3_TIMEOUT, logged_api_key=C7_API_KEY),
+ "missing_max_tokens": _gaps(dispatch=A10_NO_MAX_TOKENS),
+ "missing_api_key": _gaps(outcome=S14_MISSING_KEY, logged=S14_MISSING_KEY),
+ "stream_text": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY),
+ "stream_tool_use": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY),
+ "stream_thinking": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY),
+ "stream_error_event_mid_stream": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY),
+ "stream_longer_than_timeout": _gaps(
+ outcome=S5_STREAM_DEADLINE,
+ hidden_params=S5_STREAM_DEADLINE,
+ logged=S5_STREAM_DEADLINE,
+ logged_metadata=S5_STREAM_DEADLINE,
+ logged_api_key=S5_STREAM_DEADLINE,
+ ),
+ "stream_upstream_429": _gaps(outcome=S1_S2_ERROR_HEADERS, logged_api_key=C7_API_KEY),
+ "stream_false_explicit": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "sync_text": _gaps(
+ request=S9_SYNC,
+ outcome=S9_SYNC,
+ hidden_params=S9_SYNC,
+ logged=S9_SYNC,
+ logged_metadata=S9_SYNC,
+ logged_response_headers=S9_SYNC,
+ logged_api_key=S9_SYNC,
+ logged_call=S9_SYNC,
+ ),
+ "sync_stream": _gaps(
+ request=S9_SYNC,
+ outcome=S9_SYNC,
+ hidden_params=S9_SYNC,
+ logged=S9_SYNC,
+ logged_metadata=S9_SYNC,
+ logged_response_headers=S9_SYNC,
+ logged_api_key=S9_SYNC,
+ logged_call=S9_SYNC,
+ ),
+ "sync_upstream_429": _gaps(
+ request=S9_SYNC,
+ outcome=S9_SYNC,
+ logged=S9_SYNC,
+ logged_response_headers=S9_SYNC,
+ logged_api_key=S9_SYNC,
+ logged_call=S9_SYNC,
+ ),
+ "mock_response": _gaps(
+ request=F_MOCK,
+ outcome=F_MOCK,
+ hidden_params=F_MOCK,
+ logged=F_MOCK,
+ logged_metadata=F_MOCK,
+ logged_api_key=F_MOCK,
+ logged_call=F_MOCK,
+ ),
+ "mock_response_rate_limit": _gaps(
+ request=F_MOCK, outcome=F_MOCK, logged=F_MOCK, logged_api_key=F_MOCK, logged_call=F_MOCK
+ ),
+ "thinking_replay": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "thinking_signature_rejected_then_retried": _gaps(
+ request=F_THINKING_RETRY,
+ outcome=F_THINKING_RETRY,
+ hidden_params=F_THINKING_RETRY,
+ logged=F_THINKING_RETRY,
+ logged_metadata=C4_METADATA,
+ logged_response_headers=C6_RESPONSE_HEADERS,
+ logged_api_key=C7_API_KEY,
+ ),
+ "cache_control_injection_points": _gaps(
+ request=F_CACHE_INJECTION, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "tool_custom_field": _gaps(
+ request=F_TOOL_CUSTOM, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "strict_tools_rejected_then_retried": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY),
+ "modify_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "stream_cost_injection": _gaps(outcome=F_STREAM_COST, hidden_params=F_STREAM_COST, logged_api_key=C7_API_KEY),
+ "enable_prompt_caching": _gaps(
+ request=F_CACHE_INJECTION, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY
+ ),
+ "request_timeout_kwarg": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY),
+ "stream_timeout_kwarg": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY),
+ }
+)
+
+
+REQUEST_CASES: Final = (
+ Case("plain_text"),
+ Case("system_string", _kw(system="Answer in one word.")),
+ Case(
+ "system_blocks_with_cache_control",
+ _kw(system=[{"type": "text", "text": "Answer in one word.", "cache_control": {"type": "ephemeral"}}]),
+ ),
+ Case("multi_turn_tool_result", _kw(messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)])),
+ Case(
+ "tools_with_tool_choice",
+ _kw(tools=[dict(WEATHER_TOOL)], tool_choice={"type": "tool", "name": "get_weather"}),
+ upstream=(_ok(TOOL_USE_RESPONSE),),
+ ),
+ Case("tool_choice_none", _kw(tools=[dict(WEATHER_TOOL)], tool_choice={"type": "none"})),
+ Case("tool_input_examples", _kw(tools=[{**WEATHER_TOOL, "input_examples": [{"city": "Paris"}]}])),
+ Case("sampling_params", _kw(temperature=0.3, top_p=0.9, top_k=40)),
+ Case("stop_sequences", _kw(stop_sequences=["\n\nHuman:"])),
+ Case("metadata_user_id", _kw(metadata={"user_id": "user-123"})),
+ Case("metadata_extra_keys_are_filtered", _kw(metadata={"user_id": "user-123", "session": "s-1"})),
+ Case("metadata_user_id_not_a_string", _kw(metadata={"user_id": 123})),
+ Case("service_tier", _kw(service_tier="auto"), gaps=_gaps(request="R1: native drops service_tier")),
+ Case("thinking_enabled", _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096)),
+ Case("thinking_disabled", _kw(thinking={"type": "disabled"})),
+ Case(
+ "legacy_thinking_on_adaptive_model_keeps_display",
+ _kw(thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"}, max_tokens=4096),
+ model=OPUS_ADAPTIVE,
+ gaps=_gaps(request="R3: native replaces legacy thinking with a bare adaptive block and drops display"),
+ ),
+ Case(
+ "legacy_thinking_float_budget_on_adaptive_model",
+ _kw(thinking={"type": "enabled", "budget_tokens": 8192.0}, max_tokens=16000),
+ model=OPUS_ADAPTIVE,
+ ),
+ Case(
+ "adaptive_thinking_with_block_binding",
+ _kw(thinking={"type": "adaptive", "block_binding": "strict"}, max_tokens=4096),
+ model=OPUS_ADAPTIVE,
+ ),
+ Case("reasoning_effort", _kw(reasoning_effort="high", max_tokens=8192)),
+ Case("reasoning_effort_on_adaptive_model", _kw(reasoning_effort="low", max_tokens=8192), model=OPUS_ADAPTIVE),
+ Case("output_config_effort", _kw(output_config={"effort": "medium"}), model=OPUS_ADAPTIVE),
+ Case(
+ "output_format_json_schema",
+ _kw(
+ output_format={
+ "type": "json_schema",
+ "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
+ }
+ ),
+ ),
+ Case("speed_on_model_without_speed", _kw(speed="fast")),
+ Case("speed_on_model_without_speed_drop_params", _kw(speed="fast", drop_params=True)),
+ Case("speed_on_fast_model", _kw(speed="fast"), model=OPUS_FAST),
+ Case("temperature_on_adaptive_model", _kw(temperature=0.5), model=OPUS_ADAPTIVE),
+ Case("top_k_on_adaptive_model", _kw(top_k=5), model=OPUS_ADAPTIVE),
+ Case("top_k_on_adaptive_model_drop_params", _kw(top_k=5, drop_params=True), model=OPUS_ADAPTIVE),
+ Case(
+ "additional_drop_params_hides_rejected_temperature",
+ _kw(temperature=0.5, additional_drop_params=["temperature"]),
+ model=OPUS_ADAPTIVE,
+ ),
+ Case(
+ "additional_drop_params_nested_path",
+ _kw(
+ tools=[{**WEATHER_TOOL, "input_examples": [{"city": "Paris"}]}],
+ additional_drop_params=["tools[*].input_examples", "metadata"],
+ metadata={"user_id": "user-123"},
+ ),
+ ),
+ Case(
+ "context_management_edits",
+ _kw(context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}),
+ ),
+ Case(
+ "container_skills", _kw(container={"skills": [{"type": "anthropic", "skill_id": "pptx", "version": "latest"}]})
+ ),
+ Case(
+ "remote_mcp_servers",
+ _kw(mcp_servers=[{"type": "url", "url": "https://mcp.example.com/sse", "name": "example"}]),
+ ),
+ Case("inference_geo", _kw(inference_geo="us")),
+ Case("top_level_cache_control", _kw(cache_control={"type": "ephemeral"})),
+ Case("null_safeguards_are_stripped", _kw(safeguards=None)),
+ Case(
+ "empty_text_blocks_are_stripped",
+ _kw(
+ messages=[
+ {"role": "user", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "hi"}]},
+ ]
+ ),
+ ),
+ Case(
+ "tool_use_ids_are_sanitized",
+ _kw(
+ messages=[
+ {"role": "user", "content": "Weather?"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "tool_use", "id": "call:abc/1", "name": "get_weather", "input": dict(NO_INPUT)}
+ ],
+ },
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call:abc/1", "content": "sunny"}]},
+ ],
+ tools=[dict(WEATHER_TOOL)],
+ ),
+ ),
+ Case(
+ "provider_specific_fields_are_stripped",
+ _kw(messages=[{"role": "user", "content": "hi", "provider_specific_fields": {"x": 1}}]),
+ ),
+ Case(
+ "image_block",
+ _kw(
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_PIXEL}},
+ {"type": "text", "text": "Describe it"},
+ ],
+ }
+ ]
+ ),
+ ),
+ Case(
+ "document_block",
+ _kw(
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Paris."}},
+ {"type": "text", "text": "Summarize"},
+ ],
+ }
+ ]
+ ),
+ ),
+ Case("server_web_search_tool", _kw(tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 2}])),
+ Case("extra_headers", _kw(extra_headers={"x-trace-id": "trace-1", "anthropic-beta": "files-api-2025-04-14"})),
+ Case(
+ "anthropic_beta_from_headers_and_extra_headers",
+ _kw(headers={"anthropic-beta": "client-beta-1"}, extra_headers={"anthropic-beta": "deploy-beta-2"}),
+ gaps=_gaps(request="R2: native keeps the last anthropic-beta source instead of the union"),
+ ),
+ Case("caller_anthropic_version_capitalised", _kw(extra_headers={"Anthropic-Version": "2023-06-01"})),
+ Case(
+ "provider_specific_header",
+ _kw(provider_specific_header={"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "yes"}}),
+ ),
+ Case("explicit_custom_llm_provider", _kw(custom_llm_provider="anthropic"), model="claude-sonnet-4-5"),
+ Case("bare_model_name", model="claude-sonnet-4-5"),
+ Case("api_base_with_messages_path", _kw(api_base_suffix="/v1/messages")),
+ Case(
+ "api_base_with_messages_path_and_query",
+ _kw(api_base_suffix="/v1/messages?beta=true"),
+ ),
+ Case("api_key_from_environment", _kw(api_key=None), env=MappingProxyType({"ANTHROPIC_API_KEY": API_KEY})),
+ Case(
+ "auth_token_from_environment",
+ _kw(api_key=None),
+ env=MappingProxyType({"ANTHROPIC_AUTH_TOKEN": "oauth-token"}),
+ ),
+ Case(
+ "reasoning_auto_summary",
+ _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096),
+ env=MappingProxyType({"LITELLM_REASONING_AUTO_SUMMARY": "true"}),
+ ),
+ Case("global_drop_params", _kw(top_k=5), model=OPUS_ADAPTIVE, settings=MappingProxyType({"drop_params": True})),
+ Case("request_timeout_setting", settings=MappingProxyType({"request_timeout": 30})),
+ Case(
+ "litellm_metadata_and_metadata",
+ _kw(
+ metadata={"user_id": "user-123"},
+ litellm_metadata={"user_api_key": "hashed-key", "user_api_key_team_id": "t1"},
+ ),
+ ),
+ Case("model_info_with_id", _kw(model_info={"id": "deployment-1", "db_model": False})),
+ Case(
+ "deployment_effort_levels",
+ _kw(
+ output_config={"effort": "max"},
+ model_info={"id": "deployment-1", "reasoning_effort_levels": ["low", "high"]},
+ ),
+ model=OPUS_ADAPTIVE,
+ ),
+)
+
+RESPONSE_CASES: Final = (
+ Case("response_with_tool_use", _kw(tools=[dict(WEATHER_TOOL)]), upstream=(_ok(TOOL_USE_RESPONSE),)),
+ Case("response_with_thinking", upstream=(_ok(THINKING_RESPONSE),)),
+ Case("response_with_cache_usage", upstream=(_ok(CACHED_USAGE_RESPONSE),)),
+ Case(
+ "response_with_null_usage_and_container",
+ upstream=(_ok({**TEXT_RESPONSE, "usage": None, "container": None}),),
+ ),
+ Case(
+ "response_without_stop_sequence",
+ upstream=(_ok({k: v for k, v in TEXT_RESPONSE.items() if k != "stop_sequence"}),),
+ ),
+ Case("response_with_unknown_field", upstream=(_ok({**TEXT_RESPONSE, "future_field": {"a": 1}}),)),
+ Case("response_key_order", upstream=(_ok({"usage": TEXT_RESPONSE["usage"], **TEXT_RESPONSE}),)),
+ Case(
+ "response_upstream_headers",
+ upstream=(
+ _ok(TEXT_RESPONSE, **{"request-id": "req_01Parity", "anthropic-ratelimit-requests-remaining": "99"}),
+ ),
+ ),
+ Case("response_private_keys", private_keys=True),
+)
+
+FAILURE_CASES: Final = (
+ Case("upstream_400", upstream=(_failure(400, "invalid_request_error", "max_tokens: too large"),)),
+ Case("upstream_401", upstream=(_failure(401, "authentication_error", "invalid x-api-key"),)),
+ Case("upstream_403", upstream=(_failure(403, "permission_error", "not allowed"),)),
+ Case("upstream_404", upstream=(_failure(404, "not_found_error", "model not found"),)),
+ Case("upstream_413", upstream=(_failure(413, "request_too_large", "too large"),)),
+ Case(
+ "upstream_429_with_retry_after",
+ upstream=(
+ _failure(429, "rate_limit_error", "slow down", **{"retry-after": "7", "request-id": "req_01Parity"}),
+ ),
+ ),
+ Case("upstream_500", upstream=(_failure(500, "api_error", "internal"),)),
+ Case("upstream_529_overloaded", upstream=(_failure(529, "overloaded_error", "Overloaded"),)),
+ Case(
+ "upstream_400_context_window",
+ upstream=(_failure(400, "invalid_request_error", "prompt is too long: 250000 tokens > 200000 maximum"),),
+ ),
+ Case("upstream_non_json_error", upstream=(ResponseSpec(body=b"bad gateway", status=502),)),
+ Case("upstream_invalid_json_success", upstream=(ResponseSpec(body=b"not json", status=200),)),
+ Case("upstream_timeout", _kw(timeout=0.5), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), delay=1.5),)),
+ Case("missing_max_tokens", _kw(max_tokens=None), upstream=()),
+ Case("missing_api_key", _kw(api_key=None), upstream=()),
+)
+
+STREAM_CASES: Final = (
+ Case("stream_text", _kw(stream=True), upstream=(_stream(TEXT_STREAM),)),
+ Case("stream_tool_use", _kw(stream=True, tools=[dict(WEATHER_TOOL)]), upstream=(_stream(TOOL_STREAM),)),
+ Case(
+ "stream_thinking",
+ _kw(stream=True, thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096),
+ upstream=(_stream(THINKING_STREAM),),
+ ),
+ Case("stream_error_event_mid_stream", _kw(stream=True), upstream=(_stream(ERROR_MID_STREAM),)),
+ Case(
+ "stream_longer_than_timeout",
+ _kw(stream=True, timeout=1),
+ upstream=(_stream(SLOW_STREAM, chunk_delay=0.25),),
+ ),
+ Case(
+ "stream_upstream_429",
+ _kw(stream=True),
+ upstream=(_failure(429, "rate_limit_error", "slow down", **{"retry-after": "3"}),),
+ ),
+ Case("stream_false_explicit", _kw(stream=False)),
+)
+
+SYNC_CASES: Final = (
+ Case("sync_text", sync=True),
+ Case("sync_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM),), sync=True),
+ Case("sync_upstream_429", upstream=(_failure(429, "rate_limit_error", "slow down"),), sync=True),
+)
+
+FALLBACK_CASES: Final = (
+ Case("mock_response", _kw(mock_response="Hi from the mock"), upstream=()),
+ Case("mock_response_rate_limit", _kw(mock_response="litellm.RateLimitError"), upstream=()),
+ Case(
+ "thinking_replay",
+ _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096),
+ ),
+ Case(
+ "thinking_signature_rejected_then_retried",
+ _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096),
+ upstream=(
+ _failure(400, "invalid_request_error", "messages.1.content.0: Invalid `signature` in `thinking` block"),
+ _ok(),
+ ),
+ ),
+ Case(
+ "cache_control_injection_points",
+ _kw(system="Answer in one word.", cache_control_injection_points=[{"location": "message", "role": "system"}]),
+ ),
+ Case("tool_custom_field", _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": True}}])),
+ Case(
+ "strict_tools_rejected_then_retried",
+ _kw(tools=[{**WEATHER_TOOL, "strict": True}], drop_params=True),
+ upstream=(
+ _failure(400, "invalid_request_error", "tools.0.strict: too many strict tools"),
+ _ok(),
+ ),
+ ),
+ Case("modify_params", _kw(max_tokens=1), settings=MappingProxyType({"modify_params": True})),
+ Case(
+ "stream_cost_injection",
+ _kw(stream=True),
+ upstream=(_stream(TEXT_STREAM),),
+ settings=MappingProxyType({"include_cost_in_streaming_usage": True}),
+ ),
+ Case("enable_prompt_caching", _kw(system="Answer in one word.", enable_prompt_caching=True)),
+ Case("request_timeout_kwarg", _kw(request_timeout=30)),
+ Case("stream_timeout_kwarg", _kw(stream=True, stream_timeout=30), upstream=(_stream(TEXT_STREAM),)),
+)
+
+CASES: Final = (*REQUEST_CASES, *RESPONSE_CASES, *FAILURE_CASES, *STREAM_CASES, *SYNC_CASES, *FALLBACK_CASES)
+
+
+@dataclass(frozen=True, slots=True)
+class Leg:
+ fell_back: bool
+ requests: tuple[Mapping[str, object], ...]
+ outcome: Mapping[str, object]
+ hidden_params: Mapping[str, object]
+ logged: Mapping[str, object]
+ callback_kwargs: Mapping[str, object]
+
+
+@dataclass(slots=True)
+class FallbackProbe:
+ calls: int = 0
+
+
+_PYTHON_AMESSAGES: Final = cast(Callable[..., Awaitable[object]], dispatch._PYTHON_AMESSAGES) # pyright: ignore[reportPrivateUsage] # the Python baseline under comparison
+_PYTHON_MESSAGES: Final = cast(Callable[..., object], dispatch._PYTHON_MESSAGES) # pyright: ignore[reportPrivateUsage] # the Python baseline under comparison
+_PUBLIC_AMESSAGES: Final = cast(Callable[..., Awaitable[object]], dispatch.anthropic_messages)
+_PUBLIC_MESSAGES: Final = cast(Callable[..., object], dispatch.anthropic_messages_handler)
+
+
+@pytest.fixture
+def native_messages() -> Iterator[FallbackProbe]:
+ probe: Final = FallbackProbe()
+
+ async def python_afallback(*args: object, **kwargs: object) -> object:
+ probe.calls += 1
+ return await _PYTHON_AMESSAGES(*args, **kwargs)
+
+ def python_fallback(*args: object, **kwargs: object) -> object:
+ probe.calls += 1
+ return _PYTHON_MESSAGES(*args, **kwargs)
+
+ with (
+ rebound(catalog, "RULES", (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_OUT), *catalog.RULES)),
+ rebound(dispatch, "_PYTHON_AMESSAGES", python_afallback),
+ rebound(dispatch, "_PYTHON_MESSAGES", python_fallback),
+ ):
+ yield probe
+
+
+@contextmanager
+def _environment(env: Mapping[str, str]) -> Generator[None]:
+ # LITELLM_RUST overrides the catalog rule, so a shell that sets it would route the native leg to Python.
+ names: Final = (
+ *env,
+ "ANTHROPIC_API_KEY",
+ "ANTHROPIC_AUTH_TOKEN",
+ "LITELLM_REASONING_AUTO_SUMMARY",
+ "LITELLM_RUST",
+ )
+ with pytest.MonkeyPatch.context() as patch:
+ for name in names:
+ patch.delenv(name, raising=False)
+ for name, value in env.items():
+ patch.setenv(name, value)
+ yield
+
+
+@contextmanager
+def _settings(settings: Mapping[str, object]) -> Generator[None]:
+ saved: Final[dict[str, object]] = {name: cast(object, getattr(litellm, name)) for name in settings}
+ for name, value in settings.items():
+ setattr(litellm, name, value)
+ try:
+ yield
+ finally:
+ for name, value in saved.items():
+ setattr(litellm, name, value)
+
+
+def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]:
+ supplied: Final = dict(case.kwargs)
+ suffix: Final = supplied.pop("api_base_suffix", "")
+ kwargs: Final[dict[str, object]] = {
+ "model": case.model,
+ "max_tokens": 64,
+ "messages": [dict(USER)],
+ "api_key": API_KEY,
+ "api_base": f"{server.base_url}{suffix}",
+ **supplied,
+ }
+ return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards",)}
+
+
+def _mapping(value: object) -> Mapping[str, object]:
+ return _LOGGED.validate_python(value) if isinstance(value, dict) else {}
+
+
+def _normalized(text: str) -> str:
+ for pattern, replacement in _VOLATILE:
+ text = pattern.sub(replacement, text) # rebind-ok: successive substitutions
+ return text
+
+
+def _error(error: BaseException, phase: str, chunks: Iterable[object] = ()) -> Mapping[str, object]:
+ header_items: Final = {
+ name.lower(): value for name, value in _mapping(getattr(error, "litellm_response_headers", None)).items()
+ }
+ return {
+ "kind": "error",
+ "phase": phase,
+ "class": type(error).__name__,
+ "status_code": getattr(error, "status_code", None),
+ "llm_provider": getattr(error, "llm_provider", None),
+ "model": getattr(error, "model", None),
+ "message": _normalized(str(error)),
+ "headers": {name: header_items.get(name) for name in ("retry-after", "request-id") if name in header_items},
+ "partial": _stream_text(chunks),
+ }
+
+
+def _stream_text(chunks: Iterable[object]) -> str:
+ return b"".join(chunk if isinstance(chunk, bytes) else str(chunk).encode() for chunk in chunks).decode()
+
+
+def _response(result: object, private_keys: bool) -> Mapping[str, object]:
+ assert isinstance(result, dict), f"expected a dict response, got {type(result).__name__}"
+ body: Final = {
+ name: value
+ for name, value in cast(dict[str, object], result).items()
+ if private_keys or not name.startswith("_")
+ }
+ return {"kind": "response", "body": _OBJECT.validate_python(body)}
+
+
+def _hidden_params(result: object) -> Mapping[str, object]:
+ hidden: Final = dict(_mapping(get_hidden_params_dict(result)))
+ headers: Final = _mapping(hidden.pop("additional_headers", None))
+ header_names: Final = sorted(name for name in headers if name not in ("x-litellm-rust", "x-litellm-cache-key"))
+ return {"keys": sorted(hidden), "additional_headers": header_names}
+
+
+async def _consume_async(stream: AsyncIterator[object]) -> tuple[list[object], BaseException | None]:
+ chunks: Final[list[object]] = []
+ try:
+ async for chunk in stream:
+ chunks.append(chunk)
+ except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome
+ return chunks, error
+ return chunks, None
+
+
+def _consume_sync(stream: Iterator[object]) -> tuple[list[object], BaseException | None]:
+ chunks: Final[list[object]] = []
+ try:
+ for chunk in stream:
+ chunks.append(chunk)
+ except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome
+ return chunks, error
+ return chunks, None
+
+
+async def _invoke(
+ case: Case, kwargs: Mapping[str, object], native: bool
+) -> tuple[Mapping[str, object], Mapping[str, object]]:
+ if case.sync:
+ call = _PUBLIC_MESSAGES if native else _PYTHON_MESSAGES
+ try:
+ result = await asyncio.to_thread(lambda: call(**kwargs))
+ except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome
+ return _error(error, "call"), {}
+ if isinstance(result, dict):
+ body = cast(Mapping[str, object], result)
+ return _response(body, case.private_keys), _hidden_params(body)
+ chunks, failure = await asyncio.to_thread(_consume_sync, cast(Iterator[object], result))
+ else:
+ acall = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES
+ try:
+ result = await acall(**kwargs)
+ except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome
+ return _error(error, "call"), {}
+ if isinstance(result, dict):
+ body = cast(Mapping[str, object], result)
+ return _response(body, case.private_keys), _hidden_params(body)
+ chunks, failure = await _consume_async(cast(AsyncIterator[object], result))
+ hidden: Final = _hidden_params(result)
+ if failure is not None:
+ return _error(failure, "stream", chunks), hidden
+ return {
+ "kind": "stream",
+ "chunk_types": sorted({type(chunk).__name__ for chunk in chunks}),
+ "text": _stream_text(chunks),
+ }, hidden
+
+
+async def _terminal_events(recorder: RecordingLogger) -> tuple[str, ...]:
+ for _ in range(100):
+ await drain_logging()
+ if any(name in _TERMINAL_EVENTS for name in recorder.names):
+ break
+ await asyncio.sleep(0.05)
+ await asyncio.sleep(0.2)
+ await drain_logging()
+ return tuple(name for name in recorder.names if name in _TERMINAL_EVENTS)
+
+
+def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping[str, object], Mapping[str, object]]:
+ terminal: Final = [event for event in recorder.events if event.name in _TERMINAL_EVENTS]
+ if not terminal:
+ return {"events": events}, {}
+ details: Final = _mapping(terminal[-1].kwargs)
+ standard: Final = _mapping(details.get("standard_logging_object"))
+ error_class: Final = _mapping(standard.get("error_information")).get("error_class")
+ litellm_params: Final = _mapping(details.get("litellm_params"))
+ metadata: Final = litellm_params.get("metadata")
+ metadata_keys: Final = _mapping(metadata).keys()
+ logged: Final[Mapping[str, object]] = {
+ "events": events,
+ **{name: standard.get(name) for name in _COMPARED_LOG_FIELDS},
+ "error_class": error_class,
+ }
+ callback_kwargs: Final[Mapping[str, object]] = {
+ "logged_metadata": sorted(metadata_keys) if isinstance(metadata, dict) else metadata,
+ "logged_model_info": litellm_params.get("model_info"),
+ "logged_response_headers": bool(details.get("response_headers")),
+ "logged_api_key": details.get("api_key"),
+ "logged_call": {
+ "custom_llm_provider": details.get("custom_llm_provider"),
+ "model": details.get("model"),
+ "stream": details.get("stream"),
+ },
+ }
+ return logged, callback_kwargs
+
+
+async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg:
+ with isolated_callback_registries(), _environment(case.env), _settings(case.settings):
+ recorder: Final = RecordingLogger()
+ litellm.callbacks.append(recorder)
+ for spec in case.upstream:
+ server.enqueue(replace(spec))
+ request_count: Final = len(server.requests)
+ fallbacks_before: Final = 0 if probe is None else probe.calls
+ outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None)
+ events: Final = await _terminal_events(recorder)
+ server.responses.clear()
+ logged, callback_kwargs = _logged(recorder, events)
+ return Leg(
+ fell_back=probe is not None and probe.calls > fallbacks_before,
+ requests=tuple(
+ {
+ "method": sent.method,
+ "path": sent.path,
+ "body": sent.body if sent.body is not None else sent.raw_body.decode(errors="replace"),
+ "headers": {
+ name: value for name, value in sorted(sent.headers.items()) if name not in _TRANSPORT_HEADERS
+ },
+ }
+ for sent in server.requests[request_count:]
+ ),
+ outcome=outcome,
+ hidden_params=hidden,
+ logged=logged,
+ callback_kwargs=callback_kwargs,
+ )
+
+
+@pytest.fixture
+def parity_server(recording_server: RecordingServer) -> RecordingServer:
+ recording_server.expected_requests = None
+ return recording_server
+
+
+_LEGS: Final[dict[str, tuple[Leg, Leg]]] = {}
+
+
+async def _legs(case: Case, server: RecordingServer, probe: FallbackProbe) -> tuple[Leg, Leg]:
+ if case.name not in _LEGS:
+ _LEGS[case.name] = (await _leg(case, server, probe=None), await _leg(case, server, probe=probe))
+ _dump(case.name, *_LEGS[case.name])
+ return _LEGS[case.name]
+
+
+def _dump(name: str, python: Leg, native: Leg) -> None:
+ """Writes both legs to ``$MESSAGES_PARITY_DUMP/.json`` when that variable is set."""
+ dump: Final = os.environ.get("MESSAGES_PARITY_DUMP")
+ if not dump:
+ return
+ os.makedirs(dump, exist_ok=True)
+ with open(os.path.join(dump, f"{name}.json"), "w") as out:
+ json.dump({"python": asdict(python), "native": asdict(native)}, out, indent=1, default=repr)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("case", "aspect"), tuple(param for case in CASES for param in case.params()))
+async def test_native_messages_match_python(
+ case: Case, aspect: str, parity_server: RecordingServer, native_messages: FallbackProbe
+) -> None:
+ python, native = await _legs(case, parity_server, native_messages)
+ match aspect:
+ case "dispatch":
+ assert not native.fell_back
+ case "request":
+ assert native.requests == python.requests
+ case "outcome":
+ assert native.outcome == python.outcome
+ case "hidden_params":
+ assert native.hidden_params == python.hidden_params
+ case "logged":
+ assert native.logged == python.logged
+ case _:
+ assert native.callback_kwargs.get(aspect) == python.callback_kwargs.get(aspect)
diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py
index d7b82767b97..7beeb20f109 100644
--- a/tests/test_litellm_rust/support/recording_server.py
+++ b/tests/test_litellm_rust/support/recording_server.py
@@ -25,6 +25,7 @@ class ResponseSpec:
status: int = 200
headers: dict[str, str] = field(default_factory=dict)
delay: float = 0
+ chunk_delay: float = 0
events: tuple[tuple[str, object], ...] = ()
def payloads(self) -> tuple[bytes, ...]:
@@ -89,7 +90,9 @@ def recording_service() -> Iterator[RecordingServer]:
self.send_header(name, value)
self.end_headers()
try:
- for payload in payloads:
+ for index, payload in enumerate(payloads):
+ if index and response.chunk_delay:
+ time.sleep(response.chunk_delay)
self.wfile.write(payload)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
From 34314a99f099811348c656819841469af506dca9 Mon Sep 17 00:00:00 2001
From: Nate Armstrong
Date: Mon, 2 Nov 2026 21:02:52 -0800
Subject: [PATCH 03/25] fix(messages): make rust messages copy python messages
behaviour (#49611)
Closes the Rust vs Python /v1/messages differences the parity suite marks as known gaps, copying Python exactly.
---
.../python_contract.json | 18 +
.../callbacks-legacy-python/src/adapter.rs | 117 ++++++-
.../callbacks-legacy-python/src/callbacks.rs | 51 ++-
.../callbacks-legacy-python/src/python.rs | 6 +
.../src/test_support.rs | 7 +
litellm-rust/crates/host-http/tests/serve.rs | 14 +-
.../crates/host-native/tests/driver.rs | 8 +-
.../crates/host-python/tests/support/mod.rs | 2 +-
litellm-rust/crates/host/src/interceptors.rs | 24 +-
litellm-rust/crates/host/tests/observation.rs | 4 +-
litellm-rust/crates/http/src/error.rs | 76 ++++-
.../crates/inference-chat/src/handler.rs | 2 +-
.../crates/inference-messages/Cargo.toml | 2 +-
.../inference-messages/src/common_utils.rs | 74 ++++
.../crates/inference-messages/src/handler.rs | 191 ++++++++---
.../crates/inference-messages/src/lib.rs | 1 +
.../tests/messages/request.rs | 14 +-
.../tests/messages/response.rs | 25 +-
.../tests/messages/stream.rs | 43 +++
.../crates/inference-ocr/src/handler.rs | 4 +-
.../crates/inference-responses/src/handler.rs | 2 +-
litellm-rust/crates/inference/src/context.rs | 8 +-
litellm-rust/crates/inference/src/error.rs | 28 ++
.../src/formats/messages/request.rs | 7 +-
.../src/formats/messages/response.rs | 36 +-
.../crates/llms-types/src/recognized.rs | 10 +
.../crates/llms/src/anthropic/common_utils.rs | 57 +++-
.../llms/src/anthropic/messages/thinking.rs | 233 +++++++++++--
.../src/anthropic/messages/transformation.rs | 28 +-
.../src/azure_ai/messages/transformation.rs | 2 +-
.../llms/src/base_llm/messages/context.rs | 19 ++
litellm-rust/crates/llms/src/error.rs | 22 ++
.../src/vertex_ai/messages/transformation.rs | 2 +-
.../crates/python-bridge/src/errors.rs | 59 +++-
.../python-bridge/src/routes/messages/host.rs | 21 +-
.../python-bridge/src/routes/messages/mod.rs | 13 +-
.../host/callbacks_legacy_python.py | 45 ++-
litellm/rust_bridge/host/routes/messages.py | 200 ++++++++++-
.../sdk/messages/test_native_parity.py | 321 ++++++------------
.../cache/test_python_cache.py | 48 +--
.../messages/test_callbacks.py | 25 +-
.../messages/test_streaming.py | 13 +-
.../support/recording_server.py | 8 +-
.../rust_bridge/host/routes/test_messages.py | 107 ++++++
.../host/test_callbacks_legacy_python.py | 54 +++
.../unit/rust_bridge/messages/test_secrets.py | 12 +-
46 files changed, 1595 insertions(+), 468 deletions(-)
diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json
index f65516265c5..ab887bbc014 100644
--- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json
+++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json
@@ -33,6 +33,17 @@
"api_key",
"additional_args"
],
+ "record_response": [
+ "logger",
+ "status",
+ "headers",
+ "body"
+ ],
+ "record_api_window": [
+ "logger",
+ "start",
+ "end"
+ ],
"defers_async_logging": [
"logger"
],
@@ -91,6 +102,13 @@
"logger",
"head"
],
+ "stamp_stream_head": [
+ "head",
+ "logger",
+ "kwargs",
+ "start_time",
+ "end_time"
+ ],
"stream_success": [
"logger",
"url_route",
diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
index 9431b9d1326..1bb84184e4c 100644
--- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
+++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
@@ -57,6 +57,8 @@ pub struct LegacyLogging {
asynchronous: bool,
internal: bool,
cache_key: Option,
+ /// When the provider request went out, until its window is recorded (Messages only).
+ api_started: Option,
}
fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> {
@@ -87,6 +89,60 @@ impl LegacyLogging {
asynchronous,
internal: false,
cache_key: None,
+ api_started: None,
+ }
+ }
+
+ /// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as
+ /// its own argument, so the logger never sees the caller's raw metadata, only the validated
+ /// request metadata merged in through `litellm_params`. Python updates the logger once, before
+ /// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the
+ /// logged metadata is not rebuilt from what the call wrote into it since.
+ fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult> {
+ let kwargs = self.call.kwargs().bind(py).copy()?;
+ if self.operation != LoggingOperation::Messages {
+ return Ok(kwargs);
+ }
+ let consumed: &[&str] = if refresh {
+ &["metadata", "litellm_metadata"]
+ } else {
+ &["metadata"]
+ };
+ for name in consumed {
+ if kwargs.contains(name)? {
+ kwargs.del_item(name)?;
+ }
+ }
+ Ok(kwargs)
+ }
+
+ /// Python's Messages handler hands the logger its request params, validated `metadata`
+ /// included, as `litellm_params`, beside the `aanthropic_messages` call flag.
+ fn logged_request_params(
+ &self,
+ context: &RequestContext,
+ ) -> Option> {
+ if self.operation != LoggingOperation::Messages {
+ return None;
+ }
+ let mut params = context
+ .optional_params
+ .as_object()
+ .cloned()
+ .unwrap_or_default();
+ params.insert(
+ "aanthropic_messages".to_owned(),
+ Value::Bool(self.asynchronous),
+ );
+ Some(params)
+ }
+
+ /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync
+ /// client is untimed.
+ fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> {
+ match self.api_started.take() {
+ Some(start) => self.logger()?.record_api_window(py, start, end),
+ None => Ok(()),
}
}
@@ -111,6 +167,16 @@ impl LegacyLogging {
}
}
+ /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key.
+ fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> {
+ match self.operation {
+ LoggingOperation::Messages => Some(""),
+ LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => {
+ context.api_key.as_ref().map(|api_key| api_key.expose())
+ }
+ }
+ }
+
fn stream_billing(&self) -> Option {
match self.operation {
LoggingOperation::Messages => Some(PassThroughStream {
@@ -377,7 +443,13 @@ impl LegacyLogging {
context: &RequestContext,
) -> PyResult>> {
let logger = self.logger()?;
- logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?;
+ logger.update_from_kwargs(
+ py,
+ &self.logging_kwargs(py, false)?,
+ &wire,
+ context,
+ self.logged_request_params(context).as_ref(),
+ )?;
let body = to_py(py, &wire.body)?
.into_bound(py)
.cast_into::()?;
@@ -400,11 +472,14 @@ impl LegacyLogging {
self.logger()?.pre_call(
py,
self.input_description(),
- context.api_key.as_ref().map(|api_key| api_key.expose()),
+ self.logged_api_key(context),
&body,
&headers,
&wire.url,
)?;
+ if self.operation == LoggingOperation::Messages && self.asynchronous {
+ self.api_started = Some(epoch_seconds());
+ }
let headers = headers
.iter()
.map(|(name, value)| Ok((name.extract::()?, value.extract::()?)))
@@ -455,7 +530,7 @@ impl LegacyLogging {
py,
(
&logger,
- self.call.kwargs(),
+ self.logging_kwargs(py, true)?,
&facts.provider.model,
logger.getattr("optional_params")?,
params,
@@ -483,6 +558,16 @@ impl LegacyLogging {
py: Python<'_>,
raw: &RawResponse,
) -> PyResult> {
+ if self.operation == LoggingOperation::Messages {
+ // Python's Messages handler never calls `post_call` at receipt: it stores the raw
+ // response, and logging replays it through `AnthropicConfig.transform_response`.
+ self.close_api_window(py, epoch_seconds())?;
+ if let Some(head) = &raw.head {
+ self.logger()?
+ .record_response(py, head.status, &head.headers, &raw.body)?;
+ }
+ return Ok(HookStep::Ready(()));
+ }
let api_key = self
.request
.as_ref()
@@ -522,6 +607,9 @@ impl LegacyLogging {
) -> PyResult> {
self.end = Some(datetime(py, timing.end_time)?);
self.error = Some(error.clone_ref(py).into_value(py));
+ if self.logger.is_some() {
+ self.close_api_window(py, timing.end_time)?;
+ }
if self.stream.is_some() {
return self.stream_failure(py);
}
@@ -543,6 +631,20 @@ impl LegacyLogging {
head.bind(py).set_item("cache_key", key)?;
head.bind(py).set_item("cache_hit", true)?;
}
+ self.close_api_window(py, epoch_seconds())?;
+ if self.operation == LoggingOperation::Messages {
+ // Python's `@client` stamps the call metadata onto the stream it hands back.
+ Streaming::StampHead.call(
+ py,
+ (
+ head.bind(py),
+ self.logger()?.object(py),
+ self.call.kwargs(),
+ &self.start,
+ datetime(py, epoch_seconds())?,
+ ),
+ )?;
+ }
Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
self.stream = Some(DeliveredStream {
chunks: PyList::empty(py).unbind(),
@@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update
let step = logging
.before_provider_request(py, Box::new(wire), context)
.unwrap();
- let raw = RawResponse {
- body: "raw response".into(),
- };
+ let raw = RawResponse::new("raw response".into());
assert!(matches!(
logging
.on_event(
@@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response
py,
&locals,
c"
-assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
+assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls
assert logger.calls[0][1] is head
-chunks = logger.calls[1][1]
+assert logger.calls[1][1] is head
+chunks = logger.calls[2][1]
assert len(chunks) == 2
assert chunks[0] is first
assert chunks[1] is last
diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs
index f185f21a9c3..1951c9ebfb4 100644
--- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs
+++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs
@@ -5,6 +5,7 @@
use litellm_host::interceptors::{RequestContext, WireRequest};
use litellm_host_python::to_py;
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
+use serde_json::Value;
use crate::logger::PythonLogger;
use crate::python::{Logging, Wrapper};
@@ -15,9 +16,10 @@ pub trait LegacyCallbacks {
fn update_from_kwargs(
&self,
py: Python<'_>,
- kwargs: &Py,
+ kwargs: &Bound<'_, PyDict>,
wire: &WireRequest,
context: &RequestContext,
+ request_params: Option<&serde_json::Map>,
) -> PyResult<()>;
/// `Logging.pre_call`.
@@ -31,6 +33,20 @@ pub trait LegacyCallbacks {
url: &str,
) -> PyResult<()>;
+ /// Records the provider's raw HTTP response on the logger, as Python's Messages handler
+ /// does with `httpx_response` and `response_headers`.
+ fn record_response(
+ &self,
+ py: Python<'_>,
+ status: u16,
+ headers: &[(String, String)],
+ body: &str,
+ ) -> PyResult<()>;
+
+ /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post`
+ /// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`.
+ fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>;
+
/// `Logging.post_call`.
fn post_call(
&self,
@@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger {
fn update_from_kwargs(
&self,
py: Python<'_>,
- kwargs: &Py,
+ kwargs: &Bound<'_, PyDict>,
wire: &WireRequest,
context: &RequestContext,
+ request_params: Option<&serde_json::Map>,
) -> PyResult<()> {
let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect();
- let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?;
+ let redacted_kwargs = redact(py, kwargs, &secret_fields)?;
let optional_params = redact(
py,
&to_py(py, &context.optional_params)?
@@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger {
&secret_fields,
)?;
let params = PyDict::new(py);
- params.set_item(
- "litellm_call_id",
- kwargs.bind(py).get_item("litellm_call_id")?,
- )?;
+ params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?;
params.set_item("api_base", &wire.url)?;
for name in ["logger_fn", "litellm_request_debug"] {
- if let Some(value) = kwargs.bind(py).get_item(name)? {
+ if let Some(value) = kwargs.get_item(name)? {
params.set_item(name, value)?;
}
}
for name in custom_pricing_fields(py)? {
- if let Some(value) = kwargs.bind(py).get_item(&name)?
+ if let Some(value) = kwargs.get_item(&name)?
&& !value.is_none()
{
params.set_item(name, value)?;
}
}
+ for (name, value) in request_params.into_iter().flatten() {
+ params.set_item(name, to_py(py, value)?)?;
+ }
Logging::Update.call(
py,
(
@@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger {
Ok(())
}
+ fn record_response(
+ &self,
+ py: Python<'_>,
+ status: u16,
+ headers: &[(String, String)],
+ body: &str,
+ ) -> PyResult<()> {
+ Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?;
+ Ok(())
+ }
+
+ fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> {
+ Logging::RecordApiWindow.call(py, (self.object(py), start, end))?;
+ Ok(())
+ }
+
fn post_call(
&self,
py: Python<'_>,
diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs
index 6f1ee9dfd78..6d910d35f53 100644
--- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs
+++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs
@@ -44,6 +44,10 @@ pub(crate) enum Logging {
PreCall,
#[strum(serialize = "post_call")]
PostCall,
+ #[strum(serialize = "record_response")]
+ RecordResponse,
+ #[strum(serialize = "record_api_window")]
+ RecordApiWindow,
#[strum(serialize = "defers_async_logging")]
DefersAsync,
#[strum(serialize = "defer_success")]
@@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks {
pub(crate) enum Streaming {
#[strum(serialize = "stream_opened")]
Opened,
+ #[strum(serialize = "stamp_stream_head")]
+ StampHead,
#[strum(serialize = "stream_success")]
Success,
#[strum(serialize = "stream_failure")]
diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
index 093ce60e510..8acfa7a898b 100644
--- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
+++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
@@ -65,6 +65,12 @@ FAKES = {
'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call(
original_response, api_key, additional_args
),
+ 'record_response': lambda logger, status, headers, body: logger.model_call_details.update(
+ response_headers=dict(headers)
+ ),
+ 'record_api_window': lambda logger, start, end: logger.model_call_details.update(
+ llm_api_duration_ms=(end - start) * 1000
+ ),
'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)),
'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending),
'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls(
@@ -85,6 +91,7 @@ FAKES = {
),
'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
'stream_opened': lambda logger, head: logger.record('stream_opened', head),
+ 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head),
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
'stream_success', list(chunks)
),
diff --git a/litellm-rust/crates/host-http/tests/serve.rs b/litellm-rust/crates/host-http/tests/serve.rs
index 2ba39ad4cd2..fd1a8940b33 100644
--- a/litellm-rust/crates/host-http/tests/serve.rs
+++ b/litellm-rust/crates/host-http/tests/serve.rs
@@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept
)
.await?;
route_hooks
- .after_provider_response(RawResponse {
- body: wire.url.clone(),
- })
+ .after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(Bytes::from(wire.url)))
},
@@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once(
move |_, _, route_hooks, _observations| async move {
if event {
route_hooks
- .after_provider_response(RawResponse {
- body: "response".into(),
- })
+ .after_provider_response(RawResponse::new("response".into()))
.await?;
} else {
route_hooks
@@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto
)
.await?;
route_hooks
- .after_provider_response(RawResponse {
- body: wire.url.clone(),
- })
+ .after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(json!({"url": wire.url})))
},
@@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting(
None,
|_, _, route_hooks, _observations| async move {
route_hooks
- .after_provider_response(RawResponse { body: "raw".into() })
+ .after_provider_response(RawResponse::new("raw".into()))
.await?;
Err(TestError::Provider)
},
diff --git a/litellm-rust/crates/host-native/tests/driver.rs b/litellm-rust/crates/host-native/tests/driver.rs
index c41c1e5a43b..44c30234328 100644
--- a/litellm-rust/crates/host-native/tests/driver.rs
+++ b/litellm-rust/crates/host-native/tests/driver.rs
@@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine {
)
.await?;
route_hooks
- .after_provider_response(RawResponse {
- body: wire.url.clone(),
- })
+ .after_provider_response(RawResponse::new(wire.url.clone()))
.await?;
Ok(CallOutput::Complete(wire.url))
},
@@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re
"input",
Some(sender),
|_, _, interceptors, observers| async move {
- let raw = RawResponse {
- body: "provider response".into(),
- };
+ let raw = RawResponse::new("provider response".into());
observers.unwrap().emit(CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
));
diff --git a/litellm-rust/crates/host-python/tests/support/mod.rs b/litellm-rust/crates/host-python/tests/support/mod.rs
index 003fce74309..15e864c0718 100644
--- a/litellm-rust/crates/host-python/tests/support/mod.rs
+++ b/litellm-rust/crates/host-python/tests/support/mod.rs
@@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory {
.before_provider_request(wire(), context())
.await?;
host.interceptors
- .after_provider_response(RawResponse { body: "raw".into() })
+ .after_provider_response(RawResponse::new("raw".into()))
.await?;
Ok(format!("{projected}|{signed}|{}", wire.url))
})
diff --git a/litellm-rust/crates/host/src/interceptors.rs b/litellm-rust/crates/host/src/interceptors.rs
index 39af7c410e0..9398ea4bad6 100644
--- a/litellm-rust/crates/host/src/interceptors.rs
+++ b/litellm-rust/crates/host/src/interceptors.rs
@@ -28,6 +28,28 @@ pub struct RequestContext {
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RawResponse {
pub body: String,
+ /// The provider's HTTP status and headers, when the route forwards them to logging.
+ pub head: Option,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ResponseHead {
+ pub status: u16,
+ pub headers: Vec<(String, String)>,
+}
+
+impl RawResponse {
+ pub fn new(body: String) -> Self {
+ Self { body, head: None }
+ }
+
+ #[must_use]
+ pub fn with_head(self, head: ResponseHead) -> Self {
+ Self {
+ head: Some(head),
+ ..self
+ }
+ }
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -186,7 +208,7 @@ mod tests {
.await?;
Interceptors::after_provider_response(
&channel.interceptors,
- RawResponse { body: "raw".into() },
+ RawResponse::new("raw".into()),
)
.await?;
Ok((sent, ()))
diff --git a/litellm-rust/crates/host/tests/observation.rs b/litellm-rust/crates/host/tests/observation.rs
index 22872047d81..fba85b63bea 100644
--- a/litellm-rust/crates/host/tests/observation.rs
+++ b/litellm-rust/crates/host/tests/observation.rs
@@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile
#[rstest]
fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) {
- let mut raw = RawResponse {
- body: "provider response".into(),
- };
+ let mut raw = RawResponse::new("provider response".into());
let event: CallEvent<(), (), &RawResponse> =
CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw });
let expected = raw.clone();
diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs
index 29c8b988c0b..7078d4aba2a 100644
--- a/litellm-rust/crates/http/src/error.rs
+++ b/litellm-rust/crates/http/src/error.rs
@@ -58,8 +58,20 @@ pub(crate) enum Repr {
Network(#[source] reqwest::Error),
#[error("upstream request timed out")]
Timeout(#[source] reqwest::Error),
+ /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says
+ /// the response head had already arrived.
+ #[error("upstream request timed out")]
+ ReadTimeout {
+ #[source]
+ elapsed: Elapsed,
+ body: bool,
+ },
#[error("upstream request failed with status {status}: {body}")]
- Status { status: u16, body: UpstreamBody },
+ Status {
+ status: u16,
+ body: UpstreamBody,
+ headers: Vec<(String, String)>,
+ },
#[error("invalid WebSocket URL")]
WebSocketUrl(#[source] url::ParseError),
@@ -136,13 +148,59 @@ impl Repr {
impl Error {
pub fn upstream(status: u16, body: String) -> Self {
+ Self::upstream_with_headers(status, body, Vec::new())
+ }
+
+ /// An upstream failure that keeps the provider's response headers (`retry-after`,
+ /// `request-id`) for the caller.
+ pub fn upstream_with_headers(
+ status: u16,
+ body: String,
+ headers: Vec<(String, String)>,
+ ) -> Self {
Repr::Status {
status,
body: UpstreamBody(body),
+ headers,
}
.into()
}
+ /// The provider's response headers on an upstream failure; empty for every other error.
+ pub fn upstream_headers(&self) -> &[(String, String)] {
+ match &*self.0 {
+ Repr::Status { headers, .. } => headers,
+ _ => &[],
+ }
+ }
+
+ /// A timeout reached before the connection was established, as opposed to while reading.
+ pub fn is_connect_timeout(&self) -> bool {
+ matches!(&*self.0, Repr::Timeout(error) if error.is_connect())
+ }
+
+ /// A read the caller timed itself (httpx's per-read timeout) ran out.
+ pub fn read_timeout(elapsed: Elapsed) -> Self {
+ Repr::ReadTimeout {
+ elapsed,
+ body: false,
+ }
+ .into()
+ }
+
+ /// A read of the response body the caller timed itself ran out, after the head arrived.
+ pub fn body_read_timeout(elapsed: Elapsed) -> Self {
+ Repr::ReadTimeout {
+ elapsed,
+ body: true,
+ }
+ .into()
+ }
+
+ pub fn is_body_read_timeout(&self) -> bool {
+ matches!(&*self.0, Repr::ReadTimeout { body: true, .. })
+ }
+
pub fn computed_header(name: impl Into) -> Self {
Repr::ComputedHeader(name.into()).into()
}
@@ -169,6 +227,7 @@ impl Error {
match &*self.0 {
Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::Handshake(_)
| Repr::HandshakeTimeout(_)
@@ -227,6 +286,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::Handshake(_)
| Repr::HandshakeTimeout(_)
@@ -259,6 +319,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@@ -295,6 +356,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@@ -333,6 +395,7 @@ impl Error {
| Repr::Signature(_)
| Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
@@ -356,7 +419,10 @@ impl Error {
pub fn is_timeout(&self) -> bool {
match &*self.0 {
- Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true,
+ Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
+ | Repr::HandshakeTimeout(_)
+ | Repr::DownloadTimeout(_) => true,
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@@ -392,7 +458,7 @@ impl Error {
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match &*self.0 {
- Repr::Status { status, body } => Some((*status, &body.0)),
+ Repr::Status { status, body, .. } => Some((*status, &body.0)),
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@@ -408,6 +474,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
| Repr::WebSocketSchemeRewrite(_)
@@ -431,7 +498,7 @@ impl Error {
pub fn status(&self) -> Option {
match &*self.0 {
Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status),
- Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408),
+ Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408),
Repr::Read { .. }
| Repr::InvalidPem { .. }
| Repr::Client(_)
@@ -482,6 +549,7 @@ impl Error {
| Repr::Connect(_)
| Repr::Network(_)
| Repr::Timeout(_)
+ | Repr::ReadTimeout { .. }
| Repr::Status { .. }
| Repr::WebSocketUrl(_)
| Repr::WebSocketScheme(_)
diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs
index f948e8fffaf..cdcc3e911b9 100644
--- a/litellm-rust/crates/inference-chat/src/handler.rs
+++ b/litellm-rust/crates/inference-chat/src/handler.rs
@@ -108,7 +108,7 @@ pub(super) async fn execute(
if !status.is_success() {
return Err(litellm_http::Error::upstream(status.as_u16(), text).into());
}
- let raw = RawResponse { body: text.clone() };
+ let raw = RawResponse::new(text.clone());
if let Some(observers) = observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml
index ac52cb65a0d..0c5be52550d 100644
--- a/litellm-rust/crates/inference-messages/Cargo.toml
+++ b/litellm-rust/crates/inference-messages/Cargo.toml
@@ -24,6 +24,7 @@ litellm-tracing.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
+tokio.workspace = true
tracing.workspace = true
[dev-dependencies]
@@ -34,5 +35,4 @@ litellm-host-native.workspace = true
litellm-http = { workspace = true, features = ["test-support"] }
litellm-inference-testing.workspace = true
rstest.workspace = true
-tokio.workspace = true
wiremock.workspace = true
diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs
index a57eca1c6a9..e5ece2837d8 100644
--- a/litellm-rust/crates/inference-messages/src/common_utils.rs
+++ b/litellm-rust/crates/inference-messages/src/common_utils.rs
@@ -35,6 +35,14 @@ impl MessagesProvider {
pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig {
self.config
}
+
+ /// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped.
+ pub(crate) fn accepts_service_tier(self) -> bool {
+ matches!(
+ self.provider,
+ LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle
+ )
+ }
}
/// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
@@ -119,3 +127,69 @@ mod tests {
);
}
}
+
+/// Python's `_merge_messages_header_sources`: later sources win per header name, except
+/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and
+/// sorted into one lowercase `anthropic-beta` entry.
+pub fn merge_messages_header_sources(
+ sources: impl IntoIterator- >,
+) -> Map {
+ let mut merged = Map::new();
+ let mut betas = std::collections::BTreeSet::new();
+ for (name, value) in sources.into_iter().flatten() {
+ if !name.eq_ignore_ascii_case("anthropic-beta") {
+ merged.insert(name, value);
+ continue;
+ }
+ let values: Vec<&str> = match &value {
+ Value::String(text) => vec![text.as_str()],
+ Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(),
+ _ => Vec::new(),
+ };
+ betas.extend(
+ values
+ .iter()
+ .flat_map(|value| value.split(','))
+ .map(str::trim)
+ .filter(|beta| !beta.is_empty())
+ .map(str::to_owned),
+ );
+ }
+ if !betas.is_empty() {
+ merged.insert(
+ "anthropic-beta".to_owned(),
+ Value::String(betas.into_iter().collect::>().join(",")),
+ );
+ }
+ merged
+}
+
+#[cfg(test)]
+mod merge_tests {
+ use serde_json::{Map, Value, json};
+
+ use super::merge_messages_header_sources;
+
+ fn map(value: Value) -> Map {
+ value.as_object().cloned().unwrap()
+ }
+
+ #[test]
+ fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() {
+ let merged = merge_messages_header_sources([
+ map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})),
+ map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})),
+ map(json!({"anthropic-beta": 5})),
+ ]);
+ assert_eq!(
+ Value::Object(merged),
+ json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"})
+ );
+ }
+
+ #[test]
+ fn no_beta_values_leave_no_beta_header() {
+ let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]);
+ assert!(merged.is_empty());
+ }
+}
diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs
index 2e647688a1a..e0a7aa5adb8 100644
--- a/litellm-rust/crates/inference-messages/src/handler.rs
+++ b/litellm-rust/crates/inference-messages/src/handler.rs
@@ -10,9 +10,9 @@ use litellm_core_utils::{
get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
};
use litellm_host::interceptors::{
- Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+ Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
+ ResponseHead, ResultSource, WireRequest,
};
-use litellm_http::request::with_default_headers;
use litellm_llms::base_llm::{
auth::{Authenticated, ValidatedEnvironment, resolve_auth},
messages::{
@@ -21,14 +21,16 @@ use litellm_llms::base_llm::{
transformation::BaseMessagesConfig,
},
};
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+ MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
use litellm_secrets::source::Secrets;
use litellm_tracing::ByteChunk;
use serde_json::Value;
use super::{
Error, MessagesCall, MessagesCallResponse, MessagesRoute,
- common_utils::{MessagesProvider, string_headers},
+ common_utils::{MessagesProvider, merge_messages_header_sources, string_headers},
constants::MESSAGES_TIMEOUT_SECS,
route::Messages,
types::invalid_request,
@@ -126,14 +128,18 @@ async fn provider_call(
MessagesRequest { model, ..body },
shaping.settings.reasoning_auto_summary,
)?;
+ let transform_context =
+ MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params);
+ let filtered = requested_optional_params(sanitized, provider, &transform_context)?;
let trimmed =
- without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+ without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?;
let scoped =
get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
- let forwarded = string_headers(Some(
- extra_headers.into_iter().flatten().chain(scoped).collect(),
- ))?;
+ let forwarded = string_headers(Some(merge_messages_header_sources([
+ extra_headers.unwrap_or_default(),
+ scoped,
+ ])))?;
let validated = config.validate_environment(
forwarded,
api_key.as_deref(),
@@ -144,16 +150,13 @@ async fn provider_call(
)?;
let body = config.request_body(
- config.transform_anthropic_messages_request(
- trimmed,
- &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
- )?,
+ config.transform_anthropic_messages_request(trimmed, &transform_context)?,
&validated.headers,
shaping.settings.drop_params,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
- with_default_headers(validated.headers, config.default_headers()),
+ with_python_default_headers(validated.headers, config.default_headers()),
&body,
),
auth: validated.auth,
@@ -184,7 +187,7 @@ async fn provider_call(
.before_provider_request(
WireRequest {
url,
- headers: authenticated.headers,
+ headers: joined_header_values(authenticated.headers),
body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
},
request_context,
@@ -213,6 +216,64 @@ async fn provider_call(
})
}
+/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in
+/// case into one lowercase header whose values are joined with `", "`.
+fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> {
+ let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len());
+ let mut positions: std::collections::HashMap =
+ std::collections::HashMap::with_capacity(headers.len());
+ for (name, value) in headers {
+ match positions.entry(name.to_ascii_lowercase()) {
+ std::collections::hash_map::Entry::Occupied(position) => {
+ let present = &mut joined[*position.get()].1;
+ present.push_str(", ");
+ present.push_str(&value);
+ }
+ std::collections::hash_map::Entry::Vacant(position) => {
+ position.insert(joined.len());
+ joined.push((name, value));
+ }
+ }
+ }
+ joined
+}
+
+/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is
+/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it.
+fn with_python_default_headers(
+ headers: Vec<(String, String)>,
+ defaults: &[(&str, &str)],
+) -> Vec<(String, String)> {
+ let missing: Vec<(String, String)> = defaults
+ .iter()
+ .filter(|(name, _)| !headers.iter().any(|(present, _)| present == name))
+ .map(|(name, value)| ((*name).to_string(), (*value).to_string()))
+ .collect();
+ headers.into_iter().chain(missing).collect()
+}
+
+/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier
+/// and sampling params before `additional_drop_params` and the environment check run.
+fn requested_optional_params(
+ request: MessagesRequest,
+ provider: MessagesProvider,
+ context: &MessagesTransformContext,
+) -> Result {
+ let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params(
+ request, context,
+ )?;
+ if provider.accepts_service_tier() {
+ return Ok(request);
+ }
+ Ok(MessagesRequest {
+ params: MessagesOptionalParams {
+ service_tier: None,
+ ..request.params
+ },
+ ..request
+ })
+}
+
fn without_additional_drop_params(
request: MessagesRequest,
paths: &[String],
@@ -245,6 +306,7 @@ async fn call_provider(
} = request;
let provider_name = provider.as_str();
log_request_body(provider_name, stream, &wire.body);
+ let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS));
let response = send(
http,
Authenticated {
@@ -254,10 +316,11 @@ async fn call_provider(
&wire.url,
&wire.body,
timeout,
+ stream,
)
.await?;
if !response.status().is_success() {
- return Err(provider_error(response).await);
+ return Err(provider_error(response, timeout).await);
}
let config = provider.config();
if stream {
@@ -265,11 +328,24 @@ async fn call_provider(
response,
config.stream_decoder(),
provider_name,
+ timeout,
));
}
+ let head = ResponseHead {
+ status: response.status().as_u16(),
+ headers: litellm_http::request::header_pairs(response.headers()),
+ };
+ let status = head.status;
let text = response.text().await.map_err(litellm_http::Error::from)?;
log_response_body(&text);
- context.response_received(&text).await?;
+ context
+ .raw_response_received(RawResponse::new(text.clone()).with_head(head))
+ .await?;
+ if serde_json::from_str::(&text).is_err() {
+ // Python raises `AnthropicError(status_code=, message=)` for a
+ // body it cannot parse, and the public mapping classifies it by that status.
+ return Err(litellm_http::Error::upstream(status, text).into());
+ }
decode_response(config, &identity.model, &text)
.map(|message| MessagesCallResponse::Complete(Box::new(message)))
}
@@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error {
))
}
+/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete
+/// response keeps the client's overall deadline; a stream is bounded until its head arrives and
+/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off.
async fn send(
http: &litellm_http::Client,
authenticated: Authenticated,
url: &str,
body: &Value,
- timeout: Option,
+ timeout: Duration,
+ stream: bool,
) -> Result {
- let request = outbound_request(
- authenticated,
- url,
- body,
- Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))),
- )?;
- Ok(litellm_inference::outbound::send(request, http).await?)
+ let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?;
+ if !stream {
+ return Ok(litellm_inference::outbound::send(request, http).await?);
+ }
+ tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http))
+ .await
+ .map_err(litellm_http::Error::read_timeout)?
+ .map_err(Error::from)
}
-async fn provider_error(response: reqwest::Response) -> Error {
+/// The stream's body, each read bounded by `timeout` as httpx bounds each read.
+fn timed_chunks(
+ response: reqwest::Response,
+ timeout: Duration,
+) -> impl futures_util::Stream
- > + Send + 'static {
+ futures_util::stream::try_unfold(response, move |mut response| async move {
+ let chunk = tokio::time::timeout(timeout, response.chunk())
+ .await
+ .map_err(litellm_http::Error::body_read_timeout)?
+ .map_err(litellm_http::Error::from)?;
+ Ok(chunk.map(|chunk| (chunk, response)))
+ })
+}
+
+/// The provider's error, its body read under `timeout`: a stream has no overall deadline once
+/// its head arrives, so an error body that stalls would otherwise hold the call forever.
+async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error {
let status = response.status().as_u16();
- match response.text().await {
- Ok(text) => {
+ let headers = litellm_http::request::header_pairs(response.headers());
+ match tokio::time::timeout(timeout, response.text()).await {
+ Ok(Ok(text)) => {
log_error_body(status, &text);
- litellm_http::Error::upstream(status, text).into()
+ litellm_http::Error::upstream_with_headers(status, text, headers).into()
}
- Err(error) => litellm_http::Error::from(error).into(),
+ Ok(Err(error)) => litellm_http::Error::from(error).into(),
+ Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(),
}
}
@@ -329,22 +428,22 @@ fn streaming_response(
response: reqwest::Response,
decoder: Option,
provider: &'static str,
+ timeout: Duration,
) -> MessagesCallResponse {
let headers = response
.headers()
.iter()
.filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string())))
.collect();
+ let chunks = timed_chunks(response, timeout)
+ .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk));
let chunks = match decoder {
- None => futures_util::stream::try_unfold(response, move |mut response| async move {
- let chunk = response.chunk().await.map_err(litellm_http::Error::from)?;
- Ok(chunk.map(|chunk| {
- log_chunk(provider, "provider_response", &chunk);
- (chunk, response)
- }))
- })
- .boxed(),
- Some(decode) => decoded_chunks(response, decode, provider),
+ None => chunks.map_err(Error::from).boxed(),
+ Some(decode) => decoded_chunks(
+ chunks.map_err(std::io::Error::other).boxed(),
+ decode,
+ provider,
+ ),
};
MessagesCallResponse::Stream {
head: super::route::MessagesStreamHead { headers },
@@ -353,15 +452,10 @@ fn streaming_response(
}
fn decoded_chunks(
- response: reqwest::Response,
+ bytes: ByteStream,
decode: StreamDecoder,
provider: &'static str,
) -> BoxStream<'static, Result> {
- let bytes: ByteStream = response
- .bytes_stream()
- .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk))
- .map_err(|error| std::io::Error::other(litellm_http::Error::from(error)))
- .boxed();
futures_util::stream::try_unfold(decode(bytes), move |mut events| async move {
let Some(event) = events.try_next().await? else {
return Ok(None);
@@ -964,9 +1058,12 @@ mod tests {
.send()
.await
.unwrap();
- let MessagesCallResponse::Stream { mut chunks, .. } =
- streaming_response(response, Some(streaming::event_stream), "test")
- else {
+ let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response(
+ response,
+ Some(streaming::event_stream),
+ "test",
+ Duration::from_secs(5),
+ ) else {
panic!("a streaming response returns chunks");
};
diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs
index bfefb9c63b0..745442d0314 100644
--- a/litellm-rust/crates/inference-messages/src/lib.rs
+++ b/litellm-rust/crates/inference-messages/src/lib.rs
@@ -14,6 +14,7 @@ use std::sync::Arc;
use crate::common_utils::messages_provider;
+pub use common_utils::merge_messages_header_sources;
pub use litellm_inference::Error;
pub use types::{
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
diff --git a/litellm-rust/crates/inference-messages/tests/messages/request.rs b/litellm-rust/crates/inference-messages/tests/messages/request.rs
index e53bdb8d252..216aa96fbe6 100644
--- a/litellm-rust/crates/inference-messages/tests/messages/request.rs
+++ b/litellm-rust/crates/inference-messages/tests/messages/request.rs
@@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
#[case::deepseek("deepseek")]
#[case::minimax("minimax")]
#[tokio::test]
-async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
+/// Python adds a default unless that exact lowercase name is present, then folds the casings
+/// into one header, so a capitalised caller header is sent beside the default.
+async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults(
+ call: MessagesCall,
+ #[case] provider: &str,
+) {
let upstream = upstream([message_response()]).await;
run_message(MessagesCall {
@@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
.await;
let request = only_request(&upstream).await;
- assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]);
+ assert_eq!(
+ request.header_values("anthropic-version"),
+ ["2024-01-01, 2023-06-01"]
+ );
assert_eq!(
request.header_values("content-type"),
- ["application/json; charset=utf-8"]
+ ["application/json; charset=utf-8, application/json"]
);
}
diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs
index 7aff1286858..76813fe0a34 100644
--- a/litellm-rust/crates/inference-messages/tests/messages/response.rs
+++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs
@@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case]
}
#[rstest]
-#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))]
-#[case::not_a_message(json_response(json!({"unexpected": true})))]
#[tokio::test]
-async fn an_unreadable_success_body_is_an_invalid_response(
- call: MessagesCall,
- #[case] response: ResponseTemplate,
-) {
- let upstream = upstream([response]).await;
+async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) {
+ let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await;
+
+ let error = run(MessagesCall {
+ api_key: Some("sk".into()),
+ api_base: Some(upstream.uri()),
+ ..call
+ })
+ .await
+ .expect_err("an unparseable body fails");
+
+ assert_eq!(error.upstream_response(), Some((200, "not json")));
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) {
+ let upstream = upstream([json_response(json!({"unexpected": true}))]).await;
let error = run(MessagesCall {
api_key: Some("sk".into()),
diff --git a/litellm-rust/crates/inference-messages/tests/messages/stream.rs b/litellm-rust/crates/inference-messages/tests/messages/stream.rs
index 4a8a061f764..99ba08495aa 100644
--- a/litellm-rust/crates/inference-messages/tests/messages/stream.rs
+++ b/litellm-rust/crates/inference-messages/tests/messages/stream.rs
@@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream(
.unwrap();
}
+/// Answers with an error head whose promised body never arrives.
+async fn stalling_error_upstream() -> (String, JoinHandle<()>) {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let base = format!("http://{}", listener.local_addr().unwrap());
+ let connection = tokio::spawn(async move {
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let mut request = vec![0; 4096];
+ let _ = socket.read(&mut request).await;
+ socket
+ .write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n")
+ .await
+ .unwrap();
+ let _ = socket.read_to_end(&mut Vec::new()).await;
+ });
+ (base, connection)
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) {
+ let (base, connection) = stalling_error_upstream().await;
+ let host = RecordingStreamHost::new(
+ MessagesCall {
+ timeout: Some(Duration::from_millis(300)),
+ ..streaming(call, base)
+ },
+ usize::MAX,
+ );
+
+ let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host))
+ .await
+ .expect("the stalled error body gives up within the timeout")
+ .expect_err("a stalled error body fails the call");
+
+ assert!(error.dispatched(), "{error:?}");
+ assert_eq!(error.upstream_response(), None);
+ assert!(host.seen.into_inner().unwrap().is_empty());
+ tokio::time::timeout(Duration::from_secs(5), connection)
+ .await
+ .expect("timing out closes the upstream connection")
+ .unwrap();
+}
+
#[rstest]
#[tokio::test]
async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) {
diff --git a/litellm-rust/crates/inference-ocr/src/handler.rs b/litellm-rust/crates/inference-ocr/src/handler.rs
index c126a192153..be23af60c69 100644
--- a/litellm-rust/crates/inference-ocr/src/handler.rs
+++ b/litellm-rust/crates/inference-ocr/src/handler.rs
@@ -91,9 +91,7 @@ impl> CallHooks for OcrCallHooks<'_, H> {
}
fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
- let raw = RawResponse {
- body: String::from_utf8_lossy(body).into_owned(),
- };
+ let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned());
if let Some(observers) = self.observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs
index 19092a63a14..f81a51ad192 100644
--- a/litellm-rust/crates/inference-responses/src/handler.rs
+++ b/litellm-rust/crates/inference-responses/src/handler.rs
@@ -114,7 +114,7 @@ pub(super) async fn execute(
});
}
let body = response.text().await.map_err(litellm_http::Error::from)?;
- let raw = RawResponse { body: body.clone() };
+ let raw = RawResponse::new(body.clone());
if let Some(observers) = observers {
observers.emit(litellm_host::lifecycle::CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs
index 963aa7b90c5..ef2635d24a1 100644
--- a/litellm-rust/crates/inference/src/context.rs
+++ b/litellm-rust/crates/inference/src/context.rs
@@ -42,9 +42,11 @@ impl<'a, I: Interceptors> CallContext<'a, I> {
}
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
- let raw = RawResponse {
- body: body.to_owned(),
- };
+ self.raw_response_received(RawResponse::new(body.to_owned()))
+ .await
+ }
+
+ pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> {
if let Some(observers) = &self.observers {
observers.emit(CallEvent::Execution(
ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs
index a8e6fb4e1b3..841e0c53799 100644
--- a/litellm-rust/crates/inference/src/error.rs
+++ b/litellm-rust/crates/inference/src/error.rs
@@ -146,6 +146,34 @@ impl Error {
}
}
+ /// The provider's response headers on an upstream failure; empty for every other error.
+ pub fn upstream_headers(&self) -> &[(String, String)] {
+ match self {
+ Self::Llm(error) => error.upstream_headers(),
+ Self::Http(error) => error.upstream_headers(),
+ _ => &[],
+ }
+ }
+
+ /// A read of a streamed body timed out after the response head arrived.
+ pub fn body_read_timeout(&self) -> bool {
+ match self {
+ Self::Llm(error) => error.is_body_read_timeout(),
+ Self::Http(error) => error.is_body_read_timeout(),
+ _ => false,
+ }
+ }
+
+ /// `Some(connect)` when the call timed out, where `connect` says the connection was never
+ /// established; `None` for every other error.
+ pub fn timeout(&self) -> Option {
+ match self {
+ Self::Llm(error) => error.is_timeout(),
+ Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()),
+ _ => None,
+ }
+ }
+
pub fn upstream_response(&self) -> Option<(u16, &str)> {
match self {
Self::Llm(error) => error.upstream_response(),
diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs
index 9dbfa068025..d30d21e74e7 100644
--- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs
+++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs
@@ -264,7 +264,12 @@ pub enum ThinkingDisplay {
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct EnabledThinking {
- #[serde(default, skip_serializing_if = "Option::is_none")]
+ /// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0.
+ #[serde(
+ default,
+ skip_serializing_if = "Option::is_none",
+ deserialize_with = "crate::recognized::present"
+ )]
pub budget_tokens: Option>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub display: Option>,
diff --git a/litellm-rust/crates/llms-types/src/formats/messages/response.rs b/litellm-rust/crates/llms-types/src/formats/messages/response.rs
index 9b0c5d31b3a..276a30a6cb4 100644
--- a/litellm-rust/crates/llms-types/src/formats/messages/response.rs
+++ b/litellm-rust/crates/llms-types/src/formats/messages/response.rs
@@ -10,14 +10,29 @@ pub struct MessagesResponse {
pub content: Vec,
pub stop_reason: Option,
pub stop_sequence: Option,
- #[serde(skip_serializing_if = "Option::is_none")]
+ /// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps.
+ #[serde(
+ default,
+ deserialize_with = "present_value",
+ skip_serializing_if = "Option::is_none"
+ )]
pub usage: Option,
- #[serde(skip_serializing_if = "Option::is_none")]
+ #[serde(
+ default,
+ deserialize_with = "present_value",
+ skip_serializing_if = "Option::is_none"
+ )]
pub container: Option,
#[serde(flatten)]
pub extra: Map,
}
+fn present_value<'de, D: serde::Deserializer<'de>>(
+ deserializer: D,
+) -> Result