diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 3b311c2396b..2b64e30ec6c 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -608,7 +608,6 @@ def cost_per_token(
"input_cost_per_query", decisions_model_info.get("input_cost_per_query")
)
if isinstance(request_price, (int, float)):
- # One Decisions request can answer many questions for the same flat fee.
return float(request_price), 0.0
if decisions_overrides:
return generic_cost_per_token(
diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py
index 3b73f65c917..a48e3a2a89c 100644
--- a/litellm/decisions/call.py
+++ b/litellm/decisions/call.py
@@ -494,7 +494,7 @@ def prepare_call(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or call_api_base, dynamic_api_key or api_key
)
- except ValueError as error:
+ except (ValueError, httpx.InvalidURL) as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
if resolved_api_base is None:
raise litellm.BadRequestError(
@@ -731,7 +731,9 @@ def _formatted_response(
case DecisionsRequestBody():
return ir_to_systemone_response(caller_response, caller_request)
case OpenAIDecisionRequestBody():
- return ir_to_openai_response(caller_response, caller_request, call.requested_model)
+ return ir_to_openai_response(
+ caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
+ )
case _:
assert_never(call.request)
diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py
index e8e4b77e757..6ee44a86446 100644
--- a/litellm/llms/base_llm/decisions/transformation.py
+++ b/litellm/llms/base_llm/decisions/transformation.py
@@ -523,6 +523,7 @@ class BaseDecisionsConfig(ABC):
max_total_image_bytes: int | None = None
image_media_types: frozenset[str] | None = None
inlines_remote_images: bool = False
+ openai_evidence_fields: frozenset[str] = frozenset()
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
)
diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py
index d1fc77d2ed7..29961fc3fc5 100644
--- a/litellm/llms/openai/decisions/transformation.py
+++ b/litellm/llms/openai/decisions/transformation.py
@@ -307,7 +307,10 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) ->
def ir_to_openai_response(
- response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str
+ response: DecisionsIRResponse,
+ request: DecisionsIRRequest,
+ requested_model: str,
+ evidence_fields: frozenset[str] = frozenset(),
) -> OpenAIDecisionResponse:
usage: Final = response.usage or DecisionsIRUsage()
formatted: Final = OpenAIDecisionResponse(
@@ -327,8 +330,7 @@ def ir_to_openai_response(
total_tokens=usage.input_tokens + usage.output_tokens,
),
)
- # Keep provider evidence without letting extras replace normalized fields.
- return OpenAIDecisionResponse.model_validate({**response.extra, **formatted.model_dump()})
+ return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()})
class OpenAIDecisionsConfig(BaseDecisionsConfig):
diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py
index 5c87ae19593..006d7499271 100644
--- a/litellm/llms/parallel_ai/decisions/transformation.py
+++ b/litellm/llms/parallel_ai/decisions/transformation.py
@@ -30,6 +30,7 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
path = "/v1/decisions"
api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY")
api_base_env = ("PARALLEL_AI_API_BASE",)
+ openai_evidence_fields = frozenset({"citations", "decision_id", "mode"})
def get_default_api_base(self) -> str:
return "https://api.parallel.ai"
@@ -37,21 +38,19 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
configured_base: Final = self.resolve_api_base(None)
resolved_base: Final = api_base or configured_base
+ destination: Final = self._decisions_url(resolved_base)
if api_key:
return resolved_base, api_key
- # Compare actual endpoint URLs before loading a server-managed secret.
- trusted_urls: Final = frozenset(
- httpx.URL(self.get_complete_url(base, "decision"))
- for base in (self.get_default_api_base(), configured_base)
- if base
- )
- if resolved_base and httpx.URL(self.get_complete_url(resolved_base, "decision")) not in trusted_urls:
+ if destination not in {self._decisions_url(None), self._decisions_url(configured_base)}:
raise ValueError(
"Parallel Decisions requires an explicit api_key for a custom api_base unless "
"the destination is configured in PARALLEL_AI_API_BASE"
)
return resolved_base, self.resolve_api_key(None)
+ def _decisions_url(self, api_base: str | None) -> httpx.URL:
+ return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision"))
+
def validate_environment(
self,
headers: Mapping[str, str],
@@ -83,7 +82,6 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
response: Final = _ParallelResponse.model_validate(payload)
- # Missing answers become IR refusals in the shared System One parser.
return super().parse_response(
{
**response.model_dump(),
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index bc64a4c7ef0..30ab85fa327 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -9951,8 +9951,15 @@
"type": "string"
},
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"probabilities": {
"additionalProperties": {
@@ -9970,7 +9977,6 @@
"required": [
"type",
"choice",
- "confidence",
"probabilities"
],
"title": "ChoiceAnswer",
@@ -10401,8 +10407,15 @@
"title": "Choice"
},
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"name": {
"anyOf": [
@@ -10432,8 +10445,7 @@
"required": [
"name",
"choice",
- "probabilities",
- "confidence"
+ "probabilities"
],
"title": "OpenAIChoiceAnswer",
"type": "object"
@@ -10638,8 +10650,15 @@
"OpenAIScoreAnswer": {
"properties": {
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"name": {
"anyOf": [
@@ -10673,8 +10692,7 @@
"required": [
"name",
"score",
- "probabilities",
- "confidence"
+ "probabilities"
],
"title": "OpenAIScoreAnswer",
"type": "object"
@@ -10706,8 +10724,15 @@
"additionalProperties": true,
"properties": {
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"legend": {
"additionalProperties": {
@@ -10748,7 +10773,6 @@
"required": [
"type",
"score",
- "confidence",
"legend",
"probabilities"
],
@@ -23178,8 +23202,15 @@
"type": "string"
},
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"probabilities": {
"additionalProperties": {
@@ -23197,7 +23228,6 @@
"required": [
"type",
"choice",
- "confidence",
"probabilities"
],
"title": "ChoiceAnswer",
@@ -29029,8 +29059,15 @@
"additionalProperties": true,
"properties": {
"confidence": {
- "title": "Confidence",
- "type": "number"
+ "anyOf": [
+ {
+ "type": "number"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Confidence"
},
"legend": {
"additionalProperties": {
@@ -29071,7 +29108,6 @@
"required": [
"type",
"score",
- "confidence",
"legend",
"probabilities"
],
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index 30f1021251f..01d1e72e815 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -2424,7 +2424,7 @@ class ComplexityRouter(CustomLogger):
verdict: Final = JevVerdict(
label=answer.choice,
probabilities=answer.probabilities,
- confidence=answer.confidence,
+ confidence=answer.selected_confidence,
model=model,
cost=response.classifier_cost
if config.deployment_name
@@ -2440,7 +2440,7 @@ class ComplexityRouter(CustomLogger):
score=None,
signals=(
f"{config.provider}-classifier:{tier_name}",
- f"{config.provider}-confidence={answer.confidence:.6f}",
+ f"{config.provider}-confidence={answer.selected_confidence:.6f}",
*(
f"tier-probability:{label}={probability:.6f}"
for label, probability in answer.probabilities.items()
diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py
index 87bd031490a..2b5fce91d7c 100644
--- a/litellm/router_strategy/complexity_router/jev_classifier.py
+++ b/litellm/router_strategy/complexity_router/jev_classifier.py
@@ -58,7 +58,11 @@ class JevChoiceAnswer(LiteLLMBaseModel):
type: Literal["choice"]
choice: str
probabilities: Mapping[str, JevProbability]
- confidence: JevProbability
+ confidence: JevProbability | None = None
+
+ @property
+ def selected_confidence(self) -> float:
+ return self.probabilities.get(self.choice, 0.0) if self.confidence is None else self.confidence
class JevUsage(LiteLLMBaseModel):
diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py
index eddfe578256..aa2fd999a20 100644
--- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py
+++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py
@@ -1,4 +1,5 @@
from collections.abc import Mapping
+from dataclasses import replace
from typing import Final
import pytest
@@ -106,6 +107,19 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None:
assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
+def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None:
+ ir: Final = _openai_ir(_OPENAI_REQUEST)
+ parsed: Final = replace(
+ OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir), extra={"provider": "upstream", "citations": []}
+ )
+
+ assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
+ assert ir_to_openai_response(parsed, ir, "requested", frozenset({"citations"})).model_dump(mode="json") == {
+ **_OPENAI_RESPONSE,
+ "citations": [],
+ }
+
+
def test_answers_openai_did_not_return_are_refusals() -> None:
ir: Final = _openai_ir(_OPENAI_REQUEST)
payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]}
diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py
index d6d8af2a3fe..5bc3c96fe1f 100644
--- a/tests/unit/llms/parallel_ai/decisions/test_transformation.py
+++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py
@@ -131,7 +131,9 @@ def test_all_answers_usage_and_evidence_survive_response_translation(surface: st
formatted: Final = (
ir_to_systemone_response(response, REQUEST)
if surface == "systemone"
- else ir_to_openai_response(response, REQUEST, "parallel_ai/decision")
+ else ir_to_openai_response(
+ response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
)
result: Final = formatted.model_dump(mode="json")
assert result["citations"] == PAYLOAD["citations"]
@@ -182,7 +184,9 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence:
formatted: Final = (
ir_to_systemone_response(response, REQUEST)
if surface == "systemone"
- else ir_to_openai_response(response, REQUEST, "parallel_ai/decision")
+ else ir_to_openai_response(
+ response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
)
answers: Final = formatted.model_dump(mode="json")["answers"]
values: Final = answers.values() if isinstance(answers, dict) else answers
@@ -192,13 +196,23 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence:
def test_provider_extras_cannot_replace_normalized_response_fields() -> None:
response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST)
with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}})
- formatted: Final = ir_to_openai_response(with_collision, REQUEST, "parallel_ai/decision")
+ formatted: Final = ir_to_openai_response(
+ with_collision, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
assert formatted.model == "decision"
assert len(formatted.answers) == 4
assert formatted.usage.input_tokens == 4212
assert formatted.model_dump()["citations"] == PAYLOAD["citations"]
+def test_undocumented_parallel_fields_stay_off_the_openai_response() -> None:
+ response: Final = ParallelAIDecisionsConfig().parse_response({**PAYLOAD, "internal_trace_id": "trace"}, REQUEST)
+ formatted: Final = ir_to_openai_response(
+ response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
+ )
+ assert "internal_trace_id" not in formatted.model_dump()
+
+
@pytest.mark.parametrize("tokens", [0, 100, 32000])
@pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"])
def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None:
@@ -246,11 +260,12 @@ async def test_untrusted_base_cannot_receive_environment_key(
)
sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions
async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions
+
+ async def call() -> object:
+ return await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields)
+
with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
- if use_async:
- await async_call(**connection, **fields)
- else:
- sync_call(**connection, **fields)
+ await call()
assert not route.called
await GLOBAL_LOGGING_WORKER.clear_queue()
@@ -291,6 +306,20 @@ def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch
assert key == "server-secret"
+@pytest.mark.parametrize("api_key", [None, "caller-key"])
+@pytest.mark.parametrize("api_base", ["https://api.parallel.ai\t", "https://api.parallel.ai\r\nHost: attacker.example"])
+def test_malformed_destination_is_a_bad_request(api_base: str, api_key: str | None) -> None:
+ with pytest.raises(litellm.BadRequestError):
+ litellm.decisions(
+ model="parallel_ai/decision",
+ api_base=api_base,
+ api_key=api_key,
+ input="Example",
+ questions=[{"type": "predicate", "name": "available", "instructions": "Available?"}],
+ num_retries=0,
+ )
+
+
def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")
assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == (
diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py
index ab168b2a950..054e5eb1c65 100644
--- a/tests/unit/router_strategy/test_complexity_router.py
+++ b/tests/unit/router_strategy/test_complexity_router.py
@@ -353,6 +353,38 @@ class TestComplexityRouterInit:
"tier-probability:MEDIUM=0.900000",
)
+ @pytest.mark.asyncio
+ async def test_jev_choice_without_confidence_uses_the_selected_probability(self, mock_router_instance):
+ client = _StaticJevClient(
+ JevSystemOneResponse(
+ answers={
+ "tier": JevChoiceAnswer(
+ type="choice",
+ choice="MEDIUM",
+ probabilities={"SIMPLE": 0.1, "MEDIUM": 0.9},
+ )
+ },
+ )
+ )
+ router = ComplexityRouter(
+ "test-router",
+ mock_router_instance,
+ {
+ "classifier_type": "jev",
+ "jev_classifier_config": {"api_key": "test", "timeout_ms": 100},
+ "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
+ },
+ derive_savings_baseline=False,
+ jev_client=client,
+ )
+
+ outcome = await router.aclassify("Explain this")
+
+ assert outcome.tier == ComplexityTier.MEDIUM
+ assert outcome.cause == "jev_classifier"
+ assert outcome.jev_verdict is not None
+ assert outcome.jev_verdict.confidence == 0.9
+
@pytest.mark.asyncio
async def test_jev_pre_routing_hook_exposes_routing_decision_provenance(
self, mock_router_instance, monkeypatch: pytest.MonkeyPatch
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx
index f6c5794fa5c..9c78eff1a2b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx
@@ -109,6 +109,20 @@ describe("ResponseView integration", () => {
expect(screen.getByText("378 in / 63 out")).toBeInTheDocument();
});
+ it("leaves out the confidence badge when the provider reports no confidence", () => {
+ const withoutConfidence: SystemOneResponse = {
+ ...sampleResponse,
+ answers: {
+ dept: { type: "choice", choice: "technical", confidence: null, probabilities: { technical: 1 } },
+ frustration: { type: "score", score: 2, confidence: null, probabilities: { "2": 1 } },
+ },
+ };
+
+ render(