fix(decisions): keep Parallel evidence fields off other providers and handle missing confidence

Only fields a provider names reach the OpenAI response, so existing providers keep
their base shape. A malformed api_base answers 400 on both credential paths. The
complexity router classifier and the Playground handle a provider that omits
confidence. Regenerates the lazy OpenAPI snapshot and the dashboard API types
This commit is contained in:
mateo-berri 2026-10-30 20:07:03 -07:00 • committed by devin-gupta
parent 484340cf9e
commit 47ddc5ec62
16 changed files with 185 additions and 54 deletions

View file

@ -608,7 +608,6 @@ def cost_per_token(
"input_cost_per_query", decisions_model_info.get("input_cost_per_query")
)
if isinstance(request_price, (int, float)):
# One Decisions request can answer many questions for the same flat fee.
return float(request_price), 0.0
if decisions_overrides:
return generic_cost_per_token(

View file

@ -494,7 +494,7 @@ def prepare_call(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or call_api_base, dynamic_api_key or api_key
)
except ValueError as error:
except (ValueError, httpx.InvalidURL) as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
if resolved_api_base is None:
raise litellm.BadRequestError(
@ -731,7 +731,9 @@ def _formatted_response(
case DecisionsRequestBody():
return ir_to_systemone_response(caller_response, caller_request)
case OpenAIDecisionRequestBody():
return ir_to_openai_response(caller_response, caller_request, call.requested_model)
return ir_to_openai_response(
caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields
)
case _:
assert_never(call.request)

View file

@ -523,6 +523,7 @@ class BaseDecisionsConfig(ABC):
max_total_image_bytes: int | None = None
image_media_types: frozenset[str] | None = None
inlines_remote_images: bool = False
openai_evidence_fields: frozenset[str] = frozenset()
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
)

View file

@ -307,7 +307,10 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) ->
def ir_to_openai_response(
response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str
response: DecisionsIRResponse,
request: DecisionsIRRequest,
requested_model: str,
evidence_fields: frozenset[str] = frozenset(),
) -> OpenAIDecisionResponse:
usage: Final = response.usage or DecisionsIRUsage()
formatted: Final = OpenAIDecisionResponse(
@ -327,8 +330,7 @@ def ir_to_openai_response(
total_tokens=usage.input_tokens + usage.output_tokens,
),
)
# Keep provider evidence without letting extras replace normalized fields.
return OpenAIDecisionResponse.model_validate({**response.extra, **formatted.model_dump()})
return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()})
class OpenAIDecisionsConfig(BaseDecisionsConfig):

View file

@ -30,6 +30,7 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
path = "/v1/decisions"
api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY")
api_base_env = ("PARALLEL_AI_API_BASE",)
openai_evidence_fields = frozenset({"citations", "decision_id", "mode"})
def get_default_api_base(self) -> str:
return "https://api.parallel.ai"
@ -37,21 +38,19 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
configured_base: Final = self.resolve_api_base(None)
resolved_base: Final = api_base or configured_base
destination: Final = self._decisions_url(resolved_base)
if api_key:
return resolved_base, api_key
# Compare actual endpoint URLs before loading a server-managed secret.
trusted_urls: Final = frozenset(
httpx.URL(self.get_complete_url(base, "decision"))
for base in (self.get_default_api_base(), configured_base)
if base
)
if resolved_base and httpx.URL(self.get_complete_url(resolved_base, "decision")) not in trusted_urls:
if destination not in {self._decisions_url(None), self._decisions_url(configured_base)}:
raise ValueError(
"Parallel Decisions requires an explicit api_key for a custom api_base unless "
"the destination is configured in PARALLEL_AI_API_BASE"
)
return resolved_base, self.resolve_api_key(None)
def _decisions_url(self, api_base: str | None) -> httpx.URL:
return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision"))
def validate_environment(
self,
headers: Mapping[str, str],
@ -83,7 +82,6 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
response: Final = _ParallelResponse.model_validate(payload)
# Missing answers become IR refusals in the shared System One parser.
return super().parse_response(
{
**response.model_dump(),

View file

@ -9951,8 +9951,15 @@
"type": "string"
},
"confidence": {
"title": "Confidence",
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Confidence"
},
"probabilities": {
"additionalProperties": {
@ -9970,7 +9977,6 @@
"required": [
"type",
"choice",
"confidence",
"probabilities"
],
"title": "ChoiceAnswer",
@ -10401,8 +10407,15 @@
"title": "Choice"
},
"confidence": {
"title": "Confidence",
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Confidence"
},
"name": {
"anyOf": [
@ -10432,8 +10445,7 @@
"required": [
"name",
"choice",
"probabilities",
"confidence"
"probabilities"
],
"title": "OpenAIChoiceAnswer",
"type": "object"
@ -10638,8 +10650,15 @@
"OpenAIScoreAnswer": {
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Confidence"
},
"name": {
"anyOf": [
@ -10673,8 +10692,7 @@
"required": [
"name",
"score",
"probabilities",
"confidence"
"probabilities"
],
"title": "OpenAIScoreAnswer",
"type": "object"
@ -10706,8 +10724,15 @@
"additionalProperties": true,
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Confidence"
},
"legend": {
"additionalProperties": {
@ -10748,7 +10773,6 @@
"required": [
"type",
"score",
"confidence",
"legend",
"probabilities"
],
@ -23178,8 +23202,15 @@
"type": "string"
},
"confidence": {
"title": "Confidence",
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Confidence"
},
"probabilities": {
"additionalProperties": {
@ -23197,7 +23228,6 @@
"required": [
"type",
"choice",
"confidence",
"probabilities"
],
"title": "ChoiceAnswer",
@ -29029,8 +29059,15 @@
"additionalProperties": true,
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Confidence"
},
"legend": {
"additionalProperties": {
@ -29071,7 +29108,6 @@
"required": [
"type",
"score",
"confidence",
"legend",
"probabilities"
],

View file

@ -2424,7 +2424,7 @@ class ComplexityRouter(CustomLogger):
verdict: Final = JevVerdict(
label=answer.choice,
probabilities=answer.probabilities,
confidence=answer.confidence,
confidence=answer.selected_confidence,
model=model,
cost=response.classifier_cost
if config.deployment_name
@ -2440,7 +2440,7 @@ class ComplexityRouter(CustomLogger):
score=None,
signals=(
f"{config.provider}-classifier:{tier_name}",
f"{config.provider}-confidence={answer.confidence:.6f}",
f"{config.provider}-confidence={answer.selected_confidence:.6f}",
*(
f"tier-probability:{label}={probability:.6f}"
for label, probability in answer.probabilities.items()

View file

@ -58,7 +58,11 @@ class JevChoiceAnswer(LiteLLMBaseModel):
type: Literal["choice"]
choice: str
probabilities: Mapping[str, JevProbability]
confidence: JevProbability
confidence: JevProbability | None = None
@property
def selected_confidence(self) -> float:
return self.probabilities.get(self.choice, 0.0) if self.confidence is None else self.confidence
class JevUsage(LiteLLMBaseModel):

View file

@ -1,4 +1,5 @@
from collections.abc import Mapping
from dataclasses import replace
from typing import Final
import pytest
@ -106,6 +107,19 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None:
assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None:
ir: Final = _openai_ir(_OPENAI_REQUEST)
parsed: Final = replace(
OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir), extra={"provider": "upstream", "citations": []}
)
assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
assert ir_to_openai_response(parsed, ir, "requested", frozenset({"citations"})).model_dump(mode="json") == {
**_OPENAI_RESPONSE,
"citations": [],
}
def test_answers_openai_did_not_return_are_refusals() -> None:
ir: Final = _openai_ir(_OPENAI_REQUEST)
payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]}

View file

@ -131,7 +131,9 @@ def test_all_answers_usage_and_evidence_survive_response_translation(surface: st
formatted: Final = (
ir_to_systemone_response(response, REQUEST)
if surface == "systemone"
else ir_to_openai_response(response, REQUEST, "parallel_ai/decision")
else ir_to_openai_response(
response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
)
)
result: Final = formatted.model_dump(mode="json")
assert result["citations"] == PAYLOAD["citations"]
@ -182,7 +184,9 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence:
formatted: Final = (
ir_to_systemone_response(response, REQUEST)
if surface == "systemone"
else ir_to_openai_response(response, REQUEST, "parallel_ai/decision")
else ir_to_openai_response(
response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
)
)
answers: Final = formatted.model_dump(mode="json")["answers"]
values: Final = answers.values() if isinstance(answers, dict) else answers
@ -192,13 +196,23 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence:
def test_provider_extras_cannot_replace_normalized_response_fields() -> None:
response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST)
with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}})
formatted: Final = ir_to_openai_response(with_collision, REQUEST, "parallel_ai/decision")
formatted: Final = ir_to_openai_response(
with_collision, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
)
assert formatted.model == "decision"
assert len(formatted.answers) == 4
assert formatted.usage.input_tokens == 4212
assert formatted.model_dump()["citations"] == PAYLOAD["citations"]
def test_undocumented_parallel_fields_stay_off_the_openai_response() -> None:
response: Final = ParallelAIDecisionsConfig().parse_response({**PAYLOAD, "internal_trace_id": "trace"}, REQUEST)
formatted: Final = ir_to_openai_response(
response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields
)
assert "internal_trace_id" not in formatted.model_dump()
@pytest.mark.parametrize("tokens", [0, 100, 32000])
@pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"])
def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None:
@ -246,11 +260,12 @@ async def test_untrusted_base_cannot_receive_environment_key(
)
sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions
async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions
async def call() -> object:
return await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields)
with pytest.raises(litellm.BadRequestError, match="explicit api_key"):
if use_async:
await async_call(**connection, **fields)
else:
sync_call(**connection, **fields)
await call()
assert not route.called
await GLOBAL_LOGGING_WORKER.clear_queue()
@ -291,6 +306,20 @@ def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch
assert key == "server-secret"
@pytest.mark.parametrize("api_key", [None, "caller-key"])
@pytest.mark.parametrize("api_base", ["https://api.parallel.ai\t", "https://api.parallel.ai\r\nHost: attacker.example"])
def test_malformed_destination_is_a_bad_request(api_base: str, api_key: str | None) -> None:
with pytest.raises(litellm.BadRequestError):
litellm.decisions(
model="parallel_ai/decision",
api_base=api_base,
api_key=api_key,
input="Example",
questions=[{"type": "predicate", "name": "available", "instructions": "Available?"}],
num_retries=0,
)
def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")
assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == (

View file

@ -353,6 +353,38 @@ class TestComplexityRouterInit:
"tier-probability:MEDIUM=0.900000",
)
@pytest.mark.asyncio
async def test_jev_choice_without_confidence_uses_the_selected_probability(self, mock_router_instance):
client = _StaticJevClient(
JevSystemOneResponse(
answers={
"tier": JevChoiceAnswer(
type="choice",
choice="MEDIUM",
probabilities={"SIMPLE": 0.1, "MEDIUM": 0.9},
)
},
)
)
router = ComplexityRouter(
"test-router",
mock_router_instance,
{
"classifier_type": "jev",
"jev_classifier_config": {"api_key": "test", "timeout_ms": 100},
"tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
},
derive_savings_baseline=False,
jev_client=client,
)
outcome = await router.aclassify("Explain this")
assert outcome.tier == ComplexityTier.MEDIUM
assert outcome.cause == "jev_classifier"
assert outcome.jev_verdict is not None
assert outcome.jev_verdict.confidence == 0.9
@pytest.mark.asyncio
async def test_jev_pre_routing_hook_exposes_routing_decision_provenance(
self, mock_router_instance, monkeypatch: pytest.MonkeyPatch

View file

@ -109,6 +109,20 @@ describe("ResponseView integration", () => {
expect(screen.getByText("378 in / 63 out")).toBeInTheDocument();
});
it("leaves out the confidence badge when the provider reports no confidence", () => {
const withoutConfidence: SystemOneResponse = {
...sampleResponse,
answers: {
dept: { type: "choice", choice: "technical", confidence: null, probabilities: { technical: 1 } },
frustration: { type: "score", score: 2, confidence: null, probabilities: { "2": 1 } },
},
};
render(<ResponseView response={withoutConfidence} isLoading={false} latencyMs={120} />);
expect(screen.queryByText(/% confidence/)).not.toBeInTheDocument();
});
it("selects the score level nearest a fractional calibrated score", () => {
const fractional: SystemOneResponse = {
answers: {

View file

@ -71,7 +71,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
<div className="flex flex-wrap items-center gap-2">
<span className="text-xs text-muted-foreground">Selected choice</span>
<Badge>{answer.choice}</Badge>
{answer.confidence !== undefined && (
{typeof answer.confidence === "number" && (
<Badge variant="outline">{Math.round(answer.confidence * 100)}% confidence</Badge>
)}
</div>
@ -90,7 +90,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
<div className="flex flex-wrap items-center gap-2">
<span className="text-xs text-muted-foreground">Score</span>
<Badge>{SCORE_FORMAT.format(answer.score)}</Badge>
{answer.confidence !== undefined && (
{typeof answer.confidence === "number" && (
<Badge variant="outline">{Math.round(answer.confidence * 100)}% confidence</Badge>
)}
</div>

View file

@ -62,14 +62,14 @@ const choiceAnswerFields = {
...answerFields,
type: z.literal("choice"),
choice: choiceValue,
confidence: probability,
confidence: probability.nullish(),
probabilities: z.array(z.looseObject({ value: choiceValue, probability })),
};
const scoreAnswerFields = {
...answerFields,
type: z.literal("score"),
score: z.number().finite(),
confidence: probability,
confidence: probability.nullish(),
probabilities: z.array(z.looseObject({ value: z.number().int(), label: z.string(), probability })),
};
const usageFields = {

View file

@ -125,7 +125,7 @@ const noulAnswerSchema = z.looseObject({ type: z.literal("noul"), noul: probabil
const choiceAnswerShape = {
type: z.literal("choice"),
choice: z.string(),
confidence: probability.optional(),
confidence: probability.nullish(),
probabilities,
};
const choiceAnswerSchema = z.looseObject(choiceAnswerShape);
@ -133,7 +133,7 @@ const choiceAnswerSchema = z.looseObject(choiceAnswerShape);
const scoreAnswerShape = {
type: z.literal("score"),
score: z.number().finite(),
confidence: probability.optional(),
confidence: probability.nullish(),
legend: z.record(z.string(), decisionsJson).optional(),
probabilities,
};

View file

@ -29177,7 +29177,7 @@ export interface components {
/** Choice */
choice: string;
/** Confidence */
confidence: number;
confidence?: number | null;
/** Probabilities */
probabilities: {
[key: string]: number;
@ -39901,7 +39901,7 @@ export interface components {
/** Choice */
choice: string | boolean;
/** Confidence */
confidence: number;
confidence?: number | null;
/** Name */
name: string | null;
/** Probabilities */
@ -39997,7 +39997,7 @@ export interface components {
/** OpenAIScoreAnswer */
OpenAIScoreAnswer: {
/** Confidence */
confidence: number;
confidence?: number | null;
/** Name */
name: string | null;
/** Probabilities */
@ -45237,7 +45237,7 @@ export interface components {
/** ScoreAnswer */
ScoreAnswer: {
/** Confidence */
confidence: number;
confidence?: number | null;
/** Legend */
legend: {
[key: string]: string | {