From 16805dbb145a366dfbe0e629e57df97a885700f6 Mon Sep 17 00:00:00 2001 From: devin-gupta Date: Fri, 30 Oct 2026 05:10:11 +0000 Subject: [PATCH 1/8] feat(decisions): add Parallel AI web-grounded decisions --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + litellm/cost_calculator.py | 6 + .../llms/openai/decisions/transformation.py | 4 +- .../llms/parallel_ai/decisions/__init__.py | 0 .../parallel_ai/decisions/transformation.py | 73 ++++++ ...odel_prices_and_context_window_backup.json | 9 + .../provider_create_fields.json | 28 +++ litellm/types/decisions.py | 12 +- litellm/types/utils.py | 1 + litellm/utils.py | 9 +- model_prices_and_context_window.json | 9 + provider_endpoints_support.json | 2 +- .../test_decisions_pricing.py | 15 ++ .../llms/parallel_ai/decisions/__init__.py | 0 .../decisions/test_transformation.py | 213 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 1 + .../components/provider_info_helpers.test.tsx | 11 + .../src/components/provider_info_helpers.tsx | 3 + 19 files changed, 392 insertions(+), 9 deletions(-) create mode 100644 litellm/llms/parallel_ai/decisions/__init__.py create mode 100644 litellm/llms/parallel_ai/decisions/transformation.py create mode 100644 tests/unit/llms/parallel_ai/decisions/__init__.py create mode 100644 tests/unit/llms/parallel_ai/decisions/test_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index d59edcaff89..e6e42e37677 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1755,6 +1755,9 @@ if TYPE_CHECKING: from .llms.typesafe.decisions.transformation import ( TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, ) + from .llms.parallel_ai.decisions.transformation import ( + ParallelAIDecisionsConfig as ParallelAIDecisionsConfig, + ) from .llms.openrouter.decisions.transformation import ( OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index c1f484f8d64..33f17be450d 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMRerankConfig", "PerplexityDecisionsConfig", "TypeSafeDecisionsConfig", + "ParallelAIDecisionsConfig", "NaceDecisionsConfig", "OpenRouterDecisionsConfig", "FireworksAIDecisionsConfig", @@ -738,6 +739,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"), "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"), "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ee102e82ca4..3b311c2396b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -604,6 +604,12 @@ def cost_per_token( decisions_overrides: Final = decisions_pricing_fields( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) + request_price: Final = decisions_overrides.get( + "input_cost_per_query", decisions_model_info.get("input_cost_per_query") + ) + if isinstance(request_price, (int, float)): + # One Decisions request can answer many questions for the same flat fee. + return float(request_price), 0.0 if decisions_overrides: return generic_cost_per_token( model=model, diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 4518fe6a77e..d1fc77d2ed7 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -310,7 +310,7 @@ def ir_to_openai_response( response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str ) -> OpenAIDecisionResponse: usage: Final = response.usage or DecisionsIRUsage() - return OpenAIDecisionResponse( + formatted: Final = OpenAIDecisionResponse( model=response.model or requested_model, answers=tuple( _openai_answer(question, answer) @@ -327,6 +327,8 @@ def ir_to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) + # Keep provider evidence without letting extras replace normalized fields. + return OpenAIDecisionResponse.model_validate({**response.extra, **formatted.model_dump()}) class OpenAIDecisionsConfig(BaseDecisionsConfig): diff --git a/litellm/llms/parallel_ai/decisions/__init__.py b/litellm/llms/parallel_ai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py new file mode 100644 index 00000000000..3fb2fc89374 --- /dev/null +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -0,0 +1,73 @@ +"""Parallel's web-grounded Decisions use System One questions with mode-based routing. + +Both LiteLLM Decisions routes share this adapter. Citations describe the whole +request, so they stay on the response rather than being assigned to answers. +Contract: https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions +""" + +from collections.abc import Mapping +from typing import Final + +from pydantic import BaseModel, ConfigDict + +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, ir_to_systemone_request +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + UnsupportedDecisionsRequest, +) + + +class _ParallelResponse(BaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + mode: str + answers: Mapping[str, Mapping[str, object]] + + +class ParallelAIDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY") + api_base_env = ("PARALLEL_AI_API_BASE",) + + def get_default_api_base(self) -> str: + return "https://api.parallel.ai" + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + return { + **{ + name: value + for name, value in headers.items() + if name.lower() not in {"authorization", "x-api-key", "content-type"} + }, + **({"x-api-key": api_key} if api_key is not None else {}), + "Content-Type": "application/json", + } + + def transform_decisions_request( + self, model: str, request: DecisionsIRRequest, custom_llm_provider: str + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if len(request.questions) > 100: + return UnsupportedDecisionsRequest(reason="Parallel Decisions supports at most 100 questions") + body: Final = ir_to_systemone_request(model, request) + if isinstance(body, UnsupportedDecisionsRequest): + return body + return {"mode": model, **{key: value for key, value in body.items() if key != "model"}} + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = _ParallelResponse.model_validate(payload) + # Missing answers become IR refusals in the shared System One parser. + return super().parse_response( + { + **response.model_dump(), + "model": response.mode, + "answers": {key: answer for key, answer in response.answers.items() if answer.get("type") != "refusal"}, + }, + request, + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0799a298942..7e159f92341 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46248,6 +46248,15 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "parallel_ai/fast": { + "input_cost_per_query": 0.001, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "max_input_tokens": 32000, + "mode": "evaluation", + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api" + }, "parallel_ai/search": { "input_cost_per_query": 0.005, "litellm_provider": "parallel_ai", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index df3af4ad487..918e6e79ea0 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2613,6 +2613,34 @@ ], "default_model_placeholder": "nace/drex-latest" }, + { + "provider": "ParallelAI", + "provider_display_name": "Parallel AI", + "litellm_provider": "parallel_ai", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.parallel.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "parallel_ai/fast" + }, { "provider": "NEBIUS", "provider_display_name": "Nebius", diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 33b2c677d31..3b28225f0ff 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -210,7 +210,7 @@ class NoulAnswer(LiteLLMPydanticObjectBase): class ChoiceAnswer(LiteLLMPydanticObjectBase): type: Literal["choice"] choice: str - confidence: float + confidence: float | None = None probabilities: Mapping[str, float] model_config = ConfigDict(extra="allow", frozen=True) @@ -219,7 +219,7 @@ class ChoiceAnswer(LiteLLMPydanticObjectBase): class ScoreAnswer(LiteLLMPydanticObjectBase): type: Literal["score"] score: float - confidence: float + confidence: float | None = None legend: Mapping[str, DecisionsJSON] probabilities: Mapping[str, float] @@ -431,7 +431,7 @@ class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): name: str | None choice: str | bool probabilities: tuple[OpenAIChoiceProbability, ...] - confidence: float + confidence: float | None = None model_config = ConfigDict(frozen=True) @@ -449,7 +449,7 @@ class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): name: str | None score: float probabilities: tuple[OpenAIScoreProbability, ...] - confidence: float + confidence: float | None = None model_config = ConfigDict(frozen=True) @@ -595,7 +595,7 @@ class DecisionsIRChoiceProbability: @dataclass(frozen=True, slots=True) class DecisionsIRChoiceAnswer: choice: str | bool - confidence: float + confidence: float | None probabilities: tuple[DecisionsIRChoiceProbability, ...] extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @@ -610,7 +610,7 @@ class DecisionsIRScoreProbability: @dataclass(frozen=True, slots=True) class DecisionsIRScoreAnswer: score: float - confidence: float + confidence: float | None probabilities: tuple[DecisionsIRScoreProbability, ...] extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5905aaa3380..5d2569b2b8b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4488,6 +4488,7 @@ class LlmProviders(str, Enum): CLF_AI_GATEWAY = "clf_ai_gateway" PERPLEXITY = "perplexity" TYPESAFE = "typesafe" + PARALLEL_AI = "parallel_ai" NACE = "nace" STRANDS_DECIDER = "strands_decider" LAYA = "laya" diff --git a/litellm/utils.py b/litellm/utils.py index 41a3fddf3a3..efd56ff8015 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9991,12 +9991,19 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV): + if provider in ( + LlmProviders.OPENAI, + LlmProviders.AZURE, + LlmProviders.AZURE_AI, + LlmProviders.EU_JEV, + LlmProviders.PARALLEL_AI, + ): configs: Final = { LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, + LlmProviders.PARALLEL_AI: litellm.ParallelAIDecisionsConfig, } return configs[provider]() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0799a298942..7e159f92341 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46248,6 +46248,15 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "parallel_ai/fast": { + "input_cost_per_query": 0.001, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "max_input_tokens": 32000, + "mode": "evaluation", + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api" + }, "parallel_ai/search": { "input_cost_per_query": 0.005, "litellm_provider": "parallel_ai", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d33e61026df..36c4eb777b8 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3132,7 +3132,7 @@ "batches": false, "messages": false, "interactions": false, - "systemone": false, + "systemone": true, "a2a": false, "generateContent": false } diff --git a/tests/unit/litellm_core_utils/test_decisions_pricing.py b/tests/unit/litellm_core_utils/test_decisions_pricing.py index e9814b980d1..b006850f624 100644 --- a/tests/unit/litellm_core_utils/test_decisions_pricing.py +++ b/tests/unit/litellm_core_utils/test_decisions_pricing.py @@ -62,6 +62,21 @@ def test_apply_decisions_pricing_leaves_rows_without_decisions_fields_unchanged( assert apply_decisions_pricing(plain_row) == plain_row +@pytest.mark.parametrize("rate", [0.0, 0.002]) +def test_flat_decisions_rate_overrides_catalog_without_changing_chat( + monkeypatch: pytest.MonkeyPatch, rate: float +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "gpt-6-luna", + {**litellm.model_cost["gpt-6-luna"], **_ROW, "input_cost_per_query_decisions": rate}, + ) + assert cost_calculator.cost_per_token(model="gpt-6-luna", prompt_tokens=100, call_type="decisions") == (rate, 0.0) + assert cost_calculator.cost_per_token( + model="gpt-6-luna", prompt_tokens=100, call_type="completion" + ) == pytest.approx((100 * 1e-07, 0.0)) + + _LUNA_USAGE: Final = Usage( prompt_tokens=100, completion_tokens=10, diff --git a/tests/unit/llms/parallel_ai/decisions/__init__.py b/tests/unit/llms/parallel_ai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py new file mode 100644 index 00000000000..96b29cfadec --- /dev/null +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -0,0 +1,213 @@ +import json +from collections.abc import Mapping +from dataclasses import replace +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.base_llm.decisions.transformation import ( + DecisionsResponseValidationError, + ir_to_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.llms.parallel_ai.decisions.transformation import ParallelAIDecisionsConfig +from litellm.types.decisions import ( + DecisionsIRRefusal, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +QUESTIONS: Final = { + "available": {"type": "noul", "instructions": "Is this product available?"}, + "tier": {"type": "choice", "instructions": "Which tier is offered?", "criteria": {"free": "Free", "paid": "Paid"}}, + "fit": {"type": "score", "instructions": "How well does it fit?", "criteria": ["Poor", "Good", "Excellent"]}, + "unknown": {"type": "noul", "instructions": "Is there enough evidence?"}, +} +REQUEST: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate({"state": {"product": "Example"}, "questions": QUESTIONS}) +) +PAYLOAD: Final = { + "mode": "fast", + "answers": { + "available": {"type": "noul", "noul": 0.9}, + "tier": {"type": "choice", "choice": "paid", "probabilities": {"free": 0.2, "paid": 0.8}}, + "fit": { + "type": "score", + "score": 1.7, + "legend": {"0": "Poor", "1": "Good", "2": "Excellent"}, + "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}, + }, + "unknown": {"type": "refusal"}, + }, + "usage": {"input_tokens": 4212, "output_tokens": 0}, + "citations": [{"url": "https://example.com/product", "title": "Product", "excerpts": ["Available now"]}], + "decision_id": "decisions_example", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("surface", ["decisions", "systemone"]) +async def test_sdk_sends_parallel_wire_format_and_preserves_evidence( + use_async: bool, surface: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) + connection: Final = {"model": "parallel_ai/fast", "api_key": "test-key", "timeout": 5, "num_retries": 0} + fields: Final = ( + {"state": "Example", "questions": QUESTIONS} + if surface == "systemone" + else { + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions + async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + response: Final = await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields) + sent: Final = route.calls.last.request + body: Final = json.loads(sent.content) + assert body["mode"] == "fast" + assert "model" not in body + assert body["state"] == "Example" + assert sent.headers["x-api-key"] == "test-key" + assert "authorization" not in sent.headers + assert response.model_dump()["citations"] == PAYLOAD["citations"] + assert response.model_dump()["decision_id"] == PAYLOAD["decision_id"] + await GLOBAL_LOGGING_WORKER.clear_queue() + + +def test_provider_routes_systemone_questions_to_parallel_without_a_model_field() -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("fast", LlmProviders.PARALLEL_AI) + assert config is not None + assert config.transform_decisions_request("fast", REQUEST, "parallel_ai") == { + "mode": "fast", + "state": {"product": "Example"}, + "questions": QUESTIONS, + } + assert config.get_complete_url("https://gateway.example/v1/", "fast") == "https://gateway.example/v1/decisions" + + +def test_auth_replaces_conflicting_headers_without_forwarding_bearer_credentials() -> None: + headers: Final = ParallelAIDecisionsConfig().validate_environment( + {"Authorization": "Bearer old", "X-API-Key": "old", "CONTENT-TYPE": "text/plain", "X-Trace": "test"}, + "fast", + "test-key", + {}, + ) + assert headers == {"x-api-key": "test-key", "Content-Type": "application/json", "X-Trace": "test"} + + +def test_openai_predicate_is_translated_to_a_named_noul_question() -> None: + request: Final = openai_request_to_ir( + OpenAIDecisionRequestBody.model_validate( + { + "input": "Example product", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + ) + assert ParallelAIDecisionsConfig().transform_decisions_request("fast", request, "parallel_ai") == { + "mode": "fast", + "state": "Example product", + "questions": {"available": {"type": "noul", "instructions": "Available?"}}, + } + + +@pytest.mark.parametrize("surface", ["systemone", "decisions"]) +def test_all_answers_usage_and_evidence_survive_response_translation(surface: str) -> None: + response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) + assert isinstance(response.answers[3], DecisionsIRRefusal) + formatted: Final = ( + ir_to_systemone_response(response, REQUEST) + if surface == "systemone" + else ir_to_openai_response(response, REQUEST, "parallel_ai/fast") + ) + result: Final = formatted.model_dump(mode="json") + assert result["citations"] == PAYLOAD["citations"] + assert result["decision_id"] == PAYLOAD["decision_id"] + assert result["mode"] == "fast" + assert result["usage"]["input_tokens"] == 4212 + assert result["usage"]["output_tokens"] == 0 + if surface == "systemone": + assert result["answers"]["available"]["noul"] == 0.9 + assert result["answers"]["tier"]["choice"] == "paid" + assert result["answers"]["fit"]["score"] == 1.7 + assert "unknown" not in result["answers"] + else: + assert result["answers"][3]["type"] == "refusal" + + +def test_rejects_more_than_one_hundred_questions() -> None: + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate( + { + "state": "Example", + "questions": {str(index): QUESTIONS["available"] for index in range(101)}, + } + ) + ) + assert isinstance( + ParallelAIDecisionsConfig().transform_decisions_request("fast", request, "parallel_ai"), + UnsupportedDecisionsRequest, + ) + + +@pytest.mark.parametrize("confidence", [None, 0.8]) +@pytest.mark.parametrize("surface", ["systemone", "decisions"]) +def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: float | None, surface: str) -> None: + payload: Final = { + **PAYLOAD, + "answers": {name: {**answer, "confidence": confidence} for name, answer in PAYLOAD["answers"].items()}, + } + response: Final = ParallelAIDecisionsConfig().parse_response(payload, REQUEST) + formatted: Final = ( + ir_to_systemone_response(response, REQUEST) + if surface == "systemone" + else ir_to_openai_response(response, REQUEST, "parallel_ai/fast") + ) + answers: Final = formatted.model_dump(mode="json")["answers"] + values: Final = answers.values() if isinstance(answers, dict) else answers + assert [answer["confidence"] for answer in values if answer["type"] in {"choice", "score"}] == [confidence] * 2 + + +def test_provider_extras_cannot_replace_normalized_response_fields() -> None: + response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) + with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}}) + formatted: Final = ir_to_openai_response(with_collision, REQUEST, "parallel_ai/fast") + assert formatted.model == "fast" + assert len(formatted.answers) == 4 + assert formatted.usage.input_tokens == 4212 + assert formatted.model_dump()["citations"] == PAYLOAD["citations"] + + +@pytest.mark.parametrize("tokens", [0, 100, 32000]) +@pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"]) +def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None: + rate: Final = litellm.model_cost["parallel_ai/fast"]["input_cost_per_query"] + cost: Final = litellm.cost_per_token( + model="parallel_ai/fast", + prompt_tokens=tokens, + completion_tokens=0, + call_type=call_type, + ) + assert cost == (rate, 0.0) + + +@pytest.mark.parametrize( + "payload", [{"answers": {}}, {"mode": "fast", "answers": {"available": {"type": "noul", "noul": "invalid"}}}] +) +def test_malformed_provider_responses_raise_a_provider_error(payload: Mapping[str, object]) -> None: + with pytest.raises(DecisionsResponseValidationError): + ParallelAIDecisionsConfig().transform_decisions_response( + "fast", "parallel_ai", httpx.Response(200, json=payload), REQUEST + ) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 48672c7a305..c8f6096eaca 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -486,6 +486,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: ( ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), ("Nace", "Nace", LlmProviders.NACE), + ("ParallelAI", "Parallel AI", LlmProviders.PARALLEL_AI), ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), ), ) diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 810b010f25e..b16d22e6fc3 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -230,6 +230,7 @@ describe("provider_info_helpers", () => { Providers.NLP_CLOUD, Providers.NSCALE, Providers.OVHCLOUD, + Providers.ParallelAI, Providers.PETALS, Providers.PG_VECTOR, Providers.PREDIBASE, @@ -549,6 +550,16 @@ describe("provider_info_helpers", () => { expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); }); + it("lists Parallel decision models with their provider and model placeholder", () => { + const modelMap = { + "parallel_ai/fast": { litellm_provider: "parallel_ai", mode: "evaluation" }, + }; + + expect(getProviderModels("ParallelAI", modelMap)).toEqual(["parallel_ai/fast"]); + expect(getProviderLogoAndName("parallel_ai").displayName).toBe("Parallel AI"); + expect(getPlaceholder("ParallelAI")).toBe("parallel_ai/fast"); + }); + it("lists Nace decision models and resolves the provider name and model placeholder", () => { const modelMap = { "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 1ed32c160e6..543b931988f 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -186,6 +186,7 @@ export enum Providers { Triton = "Triton", TypeSafe = "TypeSafe", Nace = "Nace", + ParallelAI = "Parallel AI", V0 = "V0", VERCEL_AI_GATEWAY = "Vercel Ai Gateway", Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", @@ -317,6 +318,7 @@ export const provider_map: Record = { Triton: "triton", TypeSafe: "typesafe", Nace: "nace", + ParallelAI: "parallel_ai", V0: "v0", VERCEL_AI_GATEWAY: "vercel_ai_gateway", Vertex_AI: "vertex_ai", @@ -500,6 +502,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Tencent]: "tencent/deepseek-v4-pro", [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Nace]: "nace/drex-latest", + [Providers.ParallelAI]: "parallel_ai/fast", [Providers.StreamLake]: "streamlake/", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", From 8cff96765ea1c8addb7a174384903b4a8bc2b66c Mon Sep 17 00:00:00 2001 From: devin-gupta Date: Fri, 30 Oct 2026 05:42:41 +0000 Subject: [PATCH 2/8] fix(decisions): name the Parallel model decision --- .../parallel_ai/decisions/transformation.py | 8 ++-- ...odel_prices_and_context_window_backup.json | 2 +- .../provider_create_fields.json | 2 +- model_prices_and_context_window.json | 2 +- .../decisions/test_transformation.py | 37 ++++++++++++------- .../components/provider_info_helpers.test.tsx | 6 +-- .../src/components/provider_info_helpers.tsx | 2 +- 7 files changed, 35 insertions(+), 24 deletions(-) diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py index 3fb2fc89374..80823f026b2 100644 --- a/litellm/llms/parallel_ai/decisions/transformation.py +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -1,4 +1,4 @@ -"""Parallel's web-grounded Decisions use System One questions with mode-based routing. +"""Parallel's decision model uses System One questions and the native fast mode. Both LiteLLM Decisions routes share this adapter. Citations describe the whole request, so they stay on the response rather than being assigned to answers. @@ -53,12 +53,14 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): def transform_decisions_request( self, model: str, request: DecisionsIRRequest, custom_llm_provider: str ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if model != "decision": + return UnsupportedDecisionsRequest(reason="Parallel Decisions supports the decision model") if len(request.questions) > 100: return UnsupportedDecisionsRequest(reason="Parallel Decisions supports at most 100 questions") body: Final = ir_to_systemone_request(model, request) if isinstance(body, UnsupportedDecisionsRequest): return body - return {"mode": model, **{key: value for key, value in body.items() if key != "model"}} + return {"mode": "fast", **{key: value for key, value in body.items() if key != "model"}} def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: response: Final = _ParallelResponse.model_validate(payload) @@ -66,7 +68,7 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): return super().parse_response( { **response.model_dump(), - "model": response.mode, + "model": "decision", "answers": {key: answer for key, answer in response.answers.items() if answer.get("type") != "refusal"}, }, request, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7e159f92341..97185550bf9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46248,7 +46248,7 @@ "supports_response_schema": true, "supports_tool_choice": false }, - "parallel_ai/fast": { + "parallel_ai/decision": { "input_cost_per_query": 0.001, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 918e6e79ea0..0dbccff2c68 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2639,7 +2639,7 @@ "default_value": null } ], - "default_model_placeholder": "parallel_ai/fast" + "default_model_placeholder": "parallel_ai/decision" }, { "provider": "NEBIUS", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7e159f92341..97185550bf9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46248,7 +46248,7 @@ "supports_response_schema": true, "supports_tool_choice": false }, - "parallel_ai/fast": { + "parallel_ai/decision": { "input_cost_per_query": 0.001, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py index 96b29cfadec..d3a4d9b3cc3 100644 --- a/tests/unit/llms/parallel_ai/decisions/test_transformation.py +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -62,7 +62,7 @@ async def test_sdk_sends_parallel_wire_format_and_preserves_evidence( monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() route: Final = respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) - connection: Final = {"model": "parallel_ai/fast", "api_key": "test-key", "timeout": 5, "num_retries": 0} + connection: Final = {"model": "parallel_ai/decision", "api_key": "test-key", "timeout": 5, "num_retries": 0} fields: Final = ( {"state": "Example", "questions": QUESTIONS} if surface == "systemone" @@ -81,26 +81,27 @@ async def test_sdk_sends_parallel_wire_format_and_preserves_evidence( assert body["state"] == "Example" assert sent.headers["x-api-key"] == "test-key" assert "authorization" not in sent.headers + assert response.model == "decision" assert response.model_dump()["citations"] == PAYLOAD["citations"] assert response.model_dump()["decision_id"] == PAYLOAD["decision_id"] await GLOBAL_LOGGING_WORKER.clear_queue() def test_provider_routes_systemone_questions_to_parallel_without_a_model_field() -> None: - config: Final = ProviderConfigManager.get_provider_decisions_config("fast", LlmProviders.PARALLEL_AI) + config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders.PARALLEL_AI) assert config is not None - assert config.transform_decisions_request("fast", REQUEST, "parallel_ai") == { + assert config.transform_decisions_request("decision", REQUEST, "parallel_ai") == { "mode": "fast", "state": {"product": "Example"}, "questions": QUESTIONS, } - assert config.get_complete_url("https://gateway.example/v1/", "fast") == "https://gateway.example/v1/decisions" + assert config.get_complete_url("https://gateway.example/v1/", "decision") == "https://gateway.example/v1/decisions" def test_auth_replaces_conflicting_headers_without_forwarding_bearer_credentials() -> None: headers: Final = ParallelAIDecisionsConfig().validate_environment( {"Authorization": "Bearer old", "X-API-Key": "old", "CONTENT-TYPE": "text/plain", "X-Trace": "test"}, - "fast", + "decision", "test-key", {}, ) @@ -116,7 +117,7 @@ def test_openai_predicate_is_translated_to_a_named_noul_question() -> None: } ) ) - assert ParallelAIDecisionsConfig().transform_decisions_request("fast", request, "parallel_ai") == { + assert ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai") == { "mode": "fast", "state": "Example product", "questions": {"available": {"type": "noul", "instructions": "Available?"}}, @@ -130,7 +131,7 @@ def test_all_answers_usage_and_evidence_survive_response_translation(surface: st formatted: Final = ( ir_to_systemone_response(response, REQUEST) if surface == "systemone" - else ir_to_openai_response(response, REQUEST, "parallel_ai/fast") + else ir_to_openai_response(response, REQUEST, "parallel_ai/decision") ) result: Final = formatted.model_dump(mode="json") assert result["citations"] == PAYLOAD["citations"] @@ -157,7 +158,15 @@ def test_rejects_more_than_one_hundred_questions() -> None: ) ) assert isinstance( - ParallelAIDecisionsConfig().transform_decisions_request("fast", request, "parallel_ai"), + ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai"), + UnsupportedDecisionsRequest, + ) + + +@pytest.mark.parametrize("model", ["fast", "unknown"]) +def test_rejects_unregistered_models_instead_of_forwarding_them_as_modes(model: str) -> None: + assert isinstance( + ParallelAIDecisionsConfig().transform_decisions_request(model, REQUEST, "parallel_ai"), UnsupportedDecisionsRequest, ) @@ -173,7 +182,7 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: formatted: Final = ( ir_to_systemone_response(response, REQUEST) if surface == "systemone" - else ir_to_openai_response(response, REQUEST, "parallel_ai/fast") + else ir_to_openai_response(response, REQUEST, "parallel_ai/decision") ) answers: Final = formatted.model_dump(mode="json")["answers"] values: Final = answers.values() if isinstance(answers, dict) else answers @@ -183,8 +192,8 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: def test_provider_extras_cannot_replace_normalized_response_fields() -> None: response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}}) - formatted: Final = ir_to_openai_response(with_collision, REQUEST, "parallel_ai/fast") - assert formatted.model == "fast" + formatted: Final = ir_to_openai_response(with_collision, REQUEST, "parallel_ai/decision") + assert formatted.model == "decision" assert len(formatted.answers) == 4 assert formatted.usage.input_tokens == 4212 assert formatted.model_dump()["citations"] == PAYLOAD["citations"] @@ -193,9 +202,9 @@ def test_provider_extras_cannot_replace_normalized_response_fields() -> None: @pytest.mark.parametrize("tokens", [0, 100, 32000]) @pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"]) def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None: - rate: Final = litellm.model_cost["parallel_ai/fast"]["input_cost_per_query"] + rate: Final = litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"] cost: Final = litellm.cost_per_token( - model="parallel_ai/fast", + model="parallel_ai/decision", prompt_tokens=tokens, completion_tokens=0, call_type=call_type, @@ -209,5 +218,5 @@ def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: s def test_malformed_provider_responses_raise_a_provider_error(payload: Mapping[str, object]) -> None: with pytest.raises(DecisionsResponseValidationError): ParallelAIDecisionsConfig().transform_decisions_response( - "fast", "parallel_ai", httpx.Response(200, json=payload), REQUEST + "decision", "parallel_ai", httpx.Response(200, json=payload), REQUEST ) diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index b16d22e6fc3..caeb0645fc9 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -552,12 +552,12 @@ describe("provider_info_helpers", () => { it("lists Parallel decision models with their provider and model placeholder", () => { const modelMap = { - "parallel_ai/fast": { litellm_provider: "parallel_ai", mode: "evaluation" }, + "parallel_ai/decision": { litellm_provider: "parallel_ai", mode: "evaluation" }, }; - expect(getProviderModels("ParallelAI", modelMap)).toEqual(["parallel_ai/fast"]); + expect(getProviderModels("ParallelAI", modelMap)).toEqual(["parallel_ai/decision"]); expect(getProviderLogoAndName("parallel_ai").displayName).toBe("Parallel AI"); - expect(getPlaceholder("ParallelAI")).toBe("parallel_ai/fast"); + expect(getPlaceholder("ParallelAI")).toBe("parallel_ai/decision"); }); it("lists Nace decision models and resolves the provider name and model placeholder", () => { diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 543b931988f..beb9641e222 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -502,7 +502,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Tencent]: "tencent/deepseek-v4-pro", [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Nace]: "nace/drex-latest", - [Providers.ParallelAI]: "parallel_ai/fast", + [Providers.ParallelAI]: "parallel_ai/decision", [Providers.StreamLake]: "streamlake/", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", From 484340cf9e233ead1bcfc66fee9306e3acd855e1 Mon Sep 17 00:00:00 2001 From: devin-gupta Date: Fri, 30 Oct 2026 22:01:15 +0000 Subject: [PATCH 3/8] fix(parallel_ai): restrict environment credentials to trusted decision endpoints --- .../parallel_ai/decisions/transformation.py | 19 +++++ .../decisions/test_transformation.py | 77 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py index 80823f026b2..5c87ae19593 100644 --- a/litellm/llms/parallel_ai/decisions/transformation.py +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -8,6 +8,7 @@ Contract: https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typ from collections.abc import Mapping from typing import Final +import httpx from pydantic import BaseModel, ConfigDict from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, ir_to_systemone_request @@ -33,6 +34,24 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): def get_default_api_base(self) -> str: return "https://api.parallel.ai" + def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]: + configured_base: Final = self.resolve_api_base(None) + resolved_base: Final = api_base or configured_base + if api_key: + return resolved_base, api_key + # Compare actual endpoint URLs before loading a server-managed secret. + trusted_urls: Final = frozenset( + httpx.URL(self.get_complete_url(base, "decision")) + for base in (self.get_default_api_base(), configured_base) + if base + ) + if resolved_base and httpx.URL(self.get_complete_url(resolved_base, "decision")) not in trusted_urls: + raise ValueError( + "Parallel Decisions requires an explicit api_key for a custom api_base unless " + "the destination is configured in PARALLEL_AI_API_BASE" + ) + return resolved_base, self.resolve_api_key(None) + def validate_environment( self, headers: Mapping[str, str], diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py index d3a4d9b3cc3..d6d8af2a3fe 100644 --- a/tests/unit/llms/parallel_ai/decisions/test_transformation.py +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -220,3 +220,80 @@ def test_malformed_provider_responses_raise_a_provider_error(payload: Mapping[st ParallelAIDecisionsConfig().transform_decisions_response( "decision", "parallel_ai", httpx.Response(200, json=payload), REQUEST ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("surface", ["decisions", "systemone"]) +@pytest.mark.parametrize("key_env", ["PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"]) +async def test_untrusted_base_cannot_receive_environment_key( + use_async: bool, surface: str, key_env: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + monkeypatch.setenv(key_env, "server-secret") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://attacker.example/v1/decisions").respond(json=PAYLOAD) + connection: Final = {"model": "parallel_ai/decision", "api_base": "https://attacker.example", "num_retries": 0} + fields: Final = ( + {"state": "Example", "questions": QUESTIONS} + if surface == "systemone" + else { + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions + async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + with pytest.raises(litellm.BadRequestError, match="explicit api_key"): + if use_async: + await async_call(**connection, **fields) + else: + sync_call(**connection, **fields) + assert not route.called + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.parametrize( + "api_base", + [ + "https://api.parallel.ai.attacker.example", + "https://api.parallel.ai@attacker.example", + "http://api.parallel.ai", + "https://api.parallel.ai:8443", + "https://attacker.example/api.parallel.ai", + "https://api.parallel.ai/other-tenant", + ], +) +def test_environment_key_rejects_untrusted_destination_variants(api_base: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + with pytest.raises(ValueError, match="explicit api_key"): + ParallelAIDecisionsConfig().resolve_connection(api_base, None) + + +@pytest.mark.parametrize("api_base", [None, "https://api.parallel.ai", "https://API.PARALLEL.AI:443/v1/"]) +def test_default_destination_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None) + assert resolved_base == (api_base or "https://api.parallel.ai") + assert key == "server-secret" + + +@pytest.mark.parametrize("api_base", [None, "https://gateway.example/parallel/v1/"]) +def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://gateway.example/parallel") + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None) + assert resolved_base == (api_base or "https://gateway.example/parallel") + assert key == "server-secret" + + +def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == ( + "https://gateway.example", + "caller-key", + ) From 47ddc5ec629693b7cc5757f689246d786c2ad903 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:07:03 -0700 Subject: [PATCH 4/8] fix(decisions): keep Parallel evidence fields off other providers and handle missing confidence Only fields a provider names reach the OpenAI response, so existing providers keep their base shape. A malformed api_base answers 400 on both credential paths. The complexity router classifier and the Playground handle a provider that omits confidence. Regenerates the lazy OpenAPI snapshot and the dashboard API types --- litellm/cost_calculator.py | 1 - litellm/decisions/call.py | 6 +- .../llms/base_llm/decisions/transformation.py | 1 + .../llms/openai/decisions/transformation.py | 8 +- .../parallel_ai/decisions/transformation.py | 14 ++-- litellm/proxy/_lazy_openapi_snapshot.json | 76 ++++++++++++++----- .../complexity_router/complexity_router.py | 4 +- .../complexity_router/jev_classifier.py | 6 +- .../test_openai_decisions_transformation.py | 14 ++++ .../decisions/test_transformation.py | 43 +++++++++-- .../router_strategy/test_complexity_router.py | 32 ++++++++ .../ResponseView.integration.test.tsx | 14 ++++ .../components/systemOneUI/ResponseView.tsx | 4 +- .../systemOneUI/lib/openAIDecisions.ts | 4 +- .../components/systemOneUI/lib/schemas.ts | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +- 16 files changed, 185 insertions(+), 54 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 3b311c2396b..2b64e30ec6c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -608,7 +608,6 @@ def cost_per_token( "input_cost_per_query", decisions_model_info.get("input_cost_per_query") ) if isinstance(request_price, (int, float)): - # One Decisions request can answer many questions for the same flat fee. return float(request_price), 0.0 if decisions_overrides: return generic_cost_per_token( diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 3b73f65c917..a48e3a2a89c 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -494,7 +494,7 @@ def prepare_call( resolved_api_base, static_api_key = provider_config.resolve_connection( dynamic_api_base or call_api_base, dynamic_api_key or api_key ) - except ValueError as error: + except (ValueError, httpx.InvalidURL) as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error if resolved_api_base is None: raise litellm.BadRequestError( @@ -731,7 +731,9 @@ def _formatted_response( case DecisionsRequestBody(): return ir_to_systemone_response(caller_response, caller_request) case OpenAIDecisionRequestBody(): - return ir_to_openai_response(caller_response, caller_request, call.requested_model) + return ir_to_openai_response( + caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields + ) case _: assert_never(call.request) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index e8e4b77e757..6ee44a86446 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -523,6 +523,7 @@ class BaseDecisionsConfig(ABC): max_total_image_bytes: int | None = None image_media_types: frozenset[str] | None = None inlines_remote_images: bool = False + openai_evidence_fields: frozenset[str] = frozenset() health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} ) diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index d1fc77d2ed7..29961fc3fc5 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -307,7 +307,10 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> def ir_to_openai_response( - response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str + response: DecisionsIRResponse, + request: DecisionsIRRequest, + requested_model: str, + evidence_fields: frozenset[str] = frozenset(), ) -> OpenAIDecisionResponse: usage: Final = response.usage or DecisionsIRUsage() formatted: Final = OpenAIDecisionResponse( @@ -327,8 +330,7 @@ def ir_to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) - # Keep provider evidence without letting extras replace normalized fields. - return OpenAIDecisionResponse.model_validate({**response.extra, **formatted.model_dump()}) + return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()}) class OpenAIDecisionsConfig(BaseDecisionsConfig): diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py index 5c87ae19593..006d7499271 100644 --- a/litellm/llms/parallel_ai/decisions/transformation.py +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -30,6 +30,7 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): path = "/v1/decisions" api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY") api_base_env = ("PARALLEL_AI_API_BASE",) + openai_evidence_fields = frozenset({"citations", "decision_id", "mode"}) def get_default_api_base(self) -> str: return "https://api.parallel.ai" @@ -37,21 +38,19 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]: configured_base: Final = self.resolve_api_base(None) resolved_base: Final = api_base or configured_base + destination: Final = self._decisions_url(resolved_base) if api_key: return resolved_base, api_key - # Compare actual endpoint URLs before loading a server-managed secret. - trusted_urls: Final = frozenset( - httpx.URL(self.get_complete_url(base, "decision")) - for base in (self.get_default_api_base(), configured_base) - if base - ) - if resolved_base and httpx.URL(self.get_complete_url(resolved_base, "decision")) not in trusted_urls: + if destination not in {self._decisions_url(None), self._decisions_url(configured_base)}: raise ValueError( "Parallel Decisions requires an explicit api_key for a custom api_base unless " "the destination is configured in PARALLEL_AI_API_BASE" ) return resolved_base, self.resolve_api_key(None) + def _decisions_url(self, api_base: str | None) -> httpx.URL: + return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision")) + def validate_environment( self, headers: Mapping[str, str], @@ -83,7 +82,6 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: response: Final = _ParallelResponse.model_validate(payload) - # Missing answers become IR refusals in the shared System One parser. return super().parse_response( { **response.model_dump(), diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index bc64a4c7ef0..30ab85fa327 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9951,8 +9951,15 @@ "type": "string" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "probabilities": { "additionalProperties": { @@ -9970,7 +9977,6 @@ "required": [ "type", "choice", - "confidence", "probabilities" ], "title": "ChoiceAnswer", @@ -10401,8 +10407,15 @@ "title": "Choice" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "name": { "anyOf": [ @@ -10432,8 +10445,7 @@ "required": [ "name", "choice", - "probabilities", - "confidence" + "probabilities" ], "title": "OpenAIChoiceAnswer", "type": "object" @@ -10638,8 +10650,15 @@ "OpenAIScoreAnswer": { "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "name": { "anyOf": [ @@ -10673,8 +10692,7 @@ "required": [ "name", "score", - "probabilities", - "confidence" + "probabilities" ], "title": "OpenAIScoreAnswer", "type": "object" @@ -10706,8 +10724,15 @@ "additionalProperties": true, "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "legend": { "additionalProperties": { @@ -10748,7 +10773,6 @@ "required": [ "type", "score", - "confidence", "legend", "probabilities" ], @@ -23178,8 +23202,15 @@ "type": "string" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "probabilities": { "additionalProperties": { @@ -23197,7 +23228,6 @@ "required": [ "type", "choice", - "confidence", "probabilities" ], "title": "ChoiceAnswer", @@ -29029,8 +29059,15 @@ "additionalProperties": true, "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "legend": { "additionalProperties": { @@ -29071,7 +29108,6 @@ "required": [ "type", "score", - "confidence", "legend", "probabilities" ], diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 30f1021251f..01d1e72e815 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2424,7 +2424,7 @@ class ComplexityRouter(CustomLogger): verdict: Final = JevVerdict( label=answer.choice, probabilities=answer.probabilities, - confidence=answer.confidence, + confidence=answer.selected_confidence, model=model, cost=response.classifier_cost if config.deployment_name @@ -2440,7 +2440,7 @@ class ComplexityRouter(CustomLogger): score=None, signals=( f"{config.provider}-classifier:{tier_name}", - f"{config.provider}-confidence={answer.confidence:.6f}", + f"{config.provider}-confidence={answer.selected_confidence:.6f}", *( f"tier-probability:{label}={probability:.6f}" for label, probability in answer.probabilities.items() diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 87bd031490a..2b5fce91d7c 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -58,7 +58,11 @@ class JevChoiceAnswer(LiteLLMBaseModel): type: Literal["choice"] choice: str probabilities: Mapping[str, JevProbability] - confidence: JevProbability + confidence: JevProbability | None = None + + @property + def selected_confidence(self) -> float: + return self.probabilities.get(self.choice, 0.0) if self.confidence is None else self.confidence class JevUsage(LiteLLMBaseModel): diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index eddfe578256..aa2fd999a20 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -1,4 +1,5 @@ from collections.abc import Mapping +from dataclasses import replace from typing import Final import pytest @@ -106,6 +107,19 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None: assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE +def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + parsed: Final = replace( + OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir), extra={"provider": "upstream", "citations": []} + ) + + assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE + assert ir_to_openai_response(parsed, ir, "requested", frozenset({"citations"})).model_dump(mode="json") == { + **_OPENAI_RESPONSE, + "citations": [], + } + + def test_answers_openai_did_not_return_are_refusals() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py index d6d8af2a3fe..5bc3c96fe1f 100644 --- a/tests/unit/llms/parallel_ai/decisions/test_transformation.py +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -131,7 +131,9 @@ def test_all_answers_usage_and_evidence_survive_response_translation(surface: st formatted: Final = ( ir_to_systemone_response(response, REQUEST) if surface == "systemone" - else ir_to_openai_response(response, REQUEST, "parallel_ai/decision") + else ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) ) result: Final = formatted.model_dump(mode="json") assert result["citations"] == PAYLOAD["citations"] @@ -182,7 +184,9 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: formatted: Final = ( ir_to_systemone_response(response, REQUEST) if surface == "systemone" - else ir_to_openai_response(response, REQUEST, "parallel_ai/decision") + else ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) ) answers: Final = formatted.model_dump(mode="json")["answers"] values: Final = answers.values() if isinstance(answers, dict) else answers @@ -192,13 +196,23 @@ def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: def test_provider_extras_cannot_replace_normalized_response_fields() -> None: response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}}) - formatted: Final = ir_to_openai_response(with_collision, REQUEST, "parallel_ai/decision") + formatted: Final = ir_to_openai_response( + with_collision, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) assert formatted.model == "decision" assert len(formatted.answers) == 4 assert formatted.usage.input_tokens == 4212 assert formatted.model_dump()["citations"] == PAYLOAD["citations"] +def test_undocumented_parallel_fields_stay_off_the_openai_response() -> None: + response: Final = ParallelAIDecisionsConfig().parse_response({**PAYLOAD, "internal_trace_id": "trace"}, REQUEST) + formatted: Final = ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + assert "internal_trace_id" not in formatted.model_dump() + + @pytest.mark.parametrize("tokens", [0, 100, 32000]) @pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"]) def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None: @@ -246,11 +260,12 @@ async def test_untrusted_base_cannot_receive_environment_key( ) sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + + async def call() -> object: + return await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields) + with pytest.raises(litellm.BadRequestError, match="explicit api_key"): - if use_async: - await async_call(**connection, **fields) - else: - sync_call(**connection, **fields) + await call() assert not route.called await GLOBAL_LOGGING_WORKER.clear_queue() @@ -291,6 +306,20 @@ def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch assert key == "server-secret" +@pytest.mark.parametrize("api_key", [None, "caller-key"]) +@pytest.mark.parametrize("api_base", ["https://api.parallel.ai\t", "https://api.parallel.ai\r\nHost: attacker.example"]) +def test_malformed_destination_is_a_bad_request(api_base: str, api_key: str | None) -> None: + with pytest.raises(litellm.BadRequestError): + litellm.decisions( + model="parallel_ai/decision", + api_base=api_base, + api_key=api_key, + input="Example", + questions=[{"type": "predicate", "name": "available", "instructions": "Available?"}], + num_retries=0, + ) + + def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == ( diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index ab168b2a950..054e5eb1c65 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -353,6 +353,38 @@ class TestComplexityRouterInit: "tier-probability:MEDIUM=0.900000", ) + @pytest.mark.asyncio + async def test_jev_choice_without_confidence_uses_the_selected_probability(self, mock_router_instance): + client = _StaticJevClient( + JevSystemOneResponse( + answers={ + "tier": JevChoiceAnswer( + type="choice", + choice="MEDIUM", + probabilities={"SIMPLE": 0.1, "MEDIUM": 0.9}, + ) + }, + ) + ) + router = ComplexityRouter( + "test-router", + mock_router_instance, + { + "classifier_type": "jev", + "jev_classifier_config": {"api_key": "test", "timeout_ms": 100}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + jev_client=client, + ) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + assert outcome.jev_verdict is not None + assert outcome.jev_verdict.confidence == 0.9 + @pytest.mark.asyncio async def test_jev_pre_routing_hook_exposes_routing_decision_provenance( self, mock_router_instance, monkeypatch: pytest.MonkeyPatch diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx index f6c5794fa5c..9c78eff1a2b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx @@ -109,6 +109,20 @@ describe("ResponseView integration", () => { expect(screen.getByText("378 in / 63 out")).toBeInTheDocument(); }); + it("leaves out the confidence badge when the provider reports no confidence", () => { + const withoutConfidence: SystemOneResponse = { + ...sampleResponse, + answers: { + dept: { type: "choice", choice: "technical", confidence: null, probabilities: { technical: 1 } }, + frustration: { type: "score", score: 2, confidence: null, probabilities: { "2": 1 } }, + }, + }; + + render(); + + expect(screen.queryByText(/% confidence/)).not.toBeInTheDocument(); + }); + it("selects the score level nearest a fractional calibrated score", () => { const fractional: SystemOneResponse = { answers: { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx index 572bf765ac1..89d998b2632 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx @@ -71,7 +71,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
Selected choice {answer.choice} - {answer.confidence !== undefined && ( + {typeof answer.confidence === "number" && ( {Math.round(answer.confidence * 100)}% confidence )}
@@ -90,7 +90,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
Score {SCORE_FORMAT.format(answer.score)} - {answer.confidence !== undefined && ( + {typeof answer.confidence === "number" && ( {Math.round(answer.confidence * 100)}% confidence )}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts index 0e06d733ea3..1b906a8d975 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts @@ -62,14 +62,14 @@ const choiceAnswerFields = { ...answerFields, type: z.literal("choice"), choice: choiceValue, - confidence: probability, + confidence: probability.nullish(), probabilities: z.array(z.looseObject({ value: choiceValue, probability })), }; const scoreAnswerFields = { ...answerFields, type: z.literal("score"), score: z.number().finite(), - confidence: probability, + confidence: probability.nullish(), probabilities: z.array(z.looseObject({ value: z.number().int(), label: z.string(), probability })), }; const usageFields = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts index 7c923fe8e8b..108da052893 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts @@ -125,7 +125,7 @@ const noulAnswerSchema = z.looseObject({ type: z.literal("noul"), noul: probabil const choiceAnswerShape = { type: z.literal("choice"), choice: z.string(), - confidence: probability.optional(), + confidence: probability.nullish(), probabilities, }; const choiceAnswerSchema = z.looseObject(choiceAnswerShape); @@ -133,7 +133,7 @@ const choiceAnswerSchema = z.looseObject(choiceAnswerShape); const scoreAnswerShape = { type: z.literal("score"), score: z.number().finite(), - confidence: probability.optional(), + confidence: probability.nullish(), legend: z.record(z.string(), decisionsJson).optional(), probabilities, }; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13a064517a4..fccf6c1b084 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29177,7 +29177,7 @@ export interface components { /** Choice */ choice: string; /** Confidence */ - confidence: number; + confidence?: number | null; /** Probabilities */ probabilities: { [key: string]: number; @@ -39901,7 +39901,7 @@ export interface components { /** Choice */ choice: string | boolean; /** Confidence */ - confidence: number; + confidence?: number | null; /** Name */ name: string | null; /** Probabilities */ @@ -39997,7 +39997,7 @@ export interface components { /** OpenAIScoreAnswer */ OpenAIScoreAnswer: { /** Confidence */ - confidence: number; + confidence?: number | null; /** Name */ name: string | null; /** Probabilities */ @@ -45237,7 +45237,7 @@ export interface components { /** ScoreAnswer */ ScoreAnswer: { /** Confidence */ - confidence: number; + confidence?: number | null; /** Legend */ legend: { [key: string]: string | { From 657117607b9a83f23d03388181a428a60f2c1a59 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:44:02 -0700 Subject: [PATCH 5/8] chore(cost-map): declare the Decisions routes on parallel_ai/decision The provider endpoint files are generated from the cost map, so the row must name both routes for the generated System One flag to hold --- litellm/model_prices_and_context_window_backup.json | 6 +++++- litellm/provider_endpoints_support_backup.json | 2 +- model_prices_and_context_window.json | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 97185550bf9..9dc022946a6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46255,7 +46255,11 @@ "litellm_provider": "parallel_ai", "max_input_tokens": 32000, "mode": "evaluation", - "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api" + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api", + "supported_endpoints": [ + "/v1/decisions", + "/v1/systemone" + ] }, "parallel_ai/search": { "input_cost_per_query": 0.005, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d33e61026df..36c4eb777b8 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3132,7 +3132,7 @@ "batches": false, "messages": false, "interactions": false, - "systemone": false, + "systemone": true, "a2a": false, "generateContent": false } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 97185550bf9..9dc022946a6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46255,7 +46255,11 @@ "litellm_provider": "parallel_ai", "max_input_tokens": 32000, "mode": "evaluation", - "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api" + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api", + "supported_endpoints": [ + "/v1/decisions", + "/v1/systemone" + ] }, "parallel_ai/search": { "input_cost_per_query": 0.005, From 54dac276d3443e79fd350f161d5e59d156239624 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:49:40 -0700 Subject: [PATCH 6/8] fix(decisions): report no classifier confidence when a provider omits it A choice answer with no confidence is a valid classifier answer, so the complexity router picks its tier and leaves classifier_confidence and the confidence signal out instead of reporting a different metric in its place. Also show the Parallel AI logo in the provider picker and add a Parallel translation fixture captured from the live API for both Decisions routes. --- .../complexity_router/complexity_router.py | 10 +- .../complexity_router/jev_classifier.py | 6 +- tests/integration/proxy_config.yaml | 5 + .../decisions/bases/parallel_ai.py | 257 ++++++++++++++++++ .../basic/test_decisions_basic_parallel_ai.py | 11 + .../router_strategy/test_complexity_router.py | 18 +- .../components/provider_info_helpers.test.tsx | 7 +- .../src/components/provider_info_helpers.tsx | 2 + 8 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 tests/integration/translation/decisions/bases/parallel_ai.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 01d1e72e815..1e45a891604 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1213,9 +1213,11 @@ def _with_classifier_forecast( forecasted_decision: Final[StandardLoggingRoutingDecision] = { **decision, "classifier_probabilities": outcome.jev_verdict.probabilities, - "classifier_confidence": outcome.jev_verdict.confidence, } - return forecasted_decision + confidence: Final = outcome.jev_verdict.confidence + return ( + forecasted_decision if confidence is None else {**forecasted_decision, "classifier_confidence": confidence} + ) if outcome.llm_v2_forecast is not None: return _with_llm_v2_forecast(decision, outcome.llm_v2_forecast) forecast: Final = outcome.capability_forecast @@ -2424,7 +2426,7 @@ class ComplexityRouter(CustomLogger): verdict: Final = JevVerdict( label=answer.choice, probabilities=answer.probabilities, - confidence=answer.selected_confidence, + confidence=answer.confidence, model=model, cost=response.classifier_cost if config.deployment_name @@ -2440,7 +2442,7 @@ class ComplexityRouter(CustomLogger): score=None, signals=( f"{config.provider}-classifier:{tier_name}", - f"{config.provider}-confidence={answer.selected_confidence:.6f}", + *(() if answer.confidence is None else (f"{config.provider}-confidence={answer.confidence:.6f}",)), *( f"tier-probability:{label}={probability:.6f}" for label, probability in answer.probabilities.items() diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 2b5fce91d7c..39b9cae424c 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -60,10 +60,6 @@ class JevChoiceAnswer(LiteLLMBaseModel): probabilities: Mapping[str, JevProbability] confidence: JevProbability | None = None - @property - def selected_confidence(self) -> float: - return self.probabilities.get(self.choice, 0.0) if self.confidence is None else self.confidence - class JevUsage(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) @@ -294,7 +290,7 @@ class HttpJevClassifierClient: class JevVerdict(NamedTuple): label: str probabilities: Mapping[str, float] - confidence: float + confidence: float | None model: str cost: float | None provider: str = "typesafe" diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 4e2cab56d27..c552e11c22a 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -587,6 +587,11 @@ model_list: model: perplexity/pplx-decider-v1-27b api_base: http://127.0.0.1:8191 api_key: synthetic-perplexity-key + - model_name: parallel_ai/decision + litellm_params: + model: parallel_ai/decision + api_base: http://127.0.0.1:8191 + api_key: synthetic-parallel-key - model_name: typesafe/jev-1.13.0 litellm_params: model: typesafe/jev-1.13.0 diff --git a/tests/integration/translation/decisions/bases/parallel_ai.py b/tests/integration/translation/decisions/bases/parallel_ai.py new file mode 100644 index 00000000000..14e9cd6fc43 --- /dev/null +++ b/tests/integration/translation/decisions/bases/parallel_ai.py @@ -0,0 +1,257 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase +from pydantic import JsonValue + +"""Provider request and reply shape from https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions. Mock reply captured live on 2026-10-31. +""" +_CITATIONS: Final[list[JsonValue]] = [ + { + "url": "https://access.redhat.com/solutions/7108129", + "title": 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg. - Red Hat Customer Portal', + "excerpts": [ + 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg.\n\nSolution Verified - Updated 2026-06-18T01:07:40+00:00 -\nEnglish\n* No translations currently exist.\n\nIssue\n\n* NFS client hangs while returning delegations.\n* Below delegation recall errors may be reported in dmesg:\nNFS: nfs4_handle_delegation_recall_error: unhandled error -512\nNFS: nfs4_handle_delegation_recall_error: unhandled error -10024\n* We have a hanging process, it\'s using data ontop of a mounted NFS export. The fcntl() system call issued by a user process has not returned, The user process processing does not finish - usually it finishes within 2 minutes.\n\nEnvironment\n\n* Red Hat Enterprise Linux (RHEL) 8, all minor releases\n* kernel:\n* rhel8.4: kernels before 4.18.0-305.194.1.el8_4 affected\n* rhel8.10: kernels before 4.18.0-553.37.1.el8_10 affected\n* NFSv4' + ], + }, + { + "url": "https://github.com/moby/buildkit/issues/2804", + "title": "GitHub cache exporter (`type=gha`) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. · Issue #2804 · moby/buildkit · GitHub", + "excerpts": [ + 'Copy link\nOpen\nOpen\nGitHub cache exporter ( type=gha ) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. #2804\nCopy link\n\nDescription\n\n@bendavies\nbendavies\nopened on Apr 13, 2022\nIssue body actions\n\nBehaviour\n\nCache operations using type=gha will start hanging after a relavatively low number of cache reads/writes.\nWe are a small team (5 devs) and are hitting these limits constantly on our main repository. Not exactly high throughput.\n\n...\n\nCache reads and writes go towards saturating the limit, and both are affected by this throttling.\n@dhadka mentions there is a limit of about "150 cache requests in 5 minutes", and seems about right: #2276 (comment)\n\nExpected behaviour\n\nI don\'t know, but the current behaviour of hanging is not great. there is no feedback as to what is hanging resulting in the user just being confused.' + ], + }, + { + "url": "https://github.com/huginn/huginn/issues/2963", + "title": "WebsiteAgent hangs on a specific url, never finishes job or times out · Issue #2963 · huginn/huginn · GitHub", + "excerpts": [ + 'WebsiteAgent hangs on a specific url, never finishes job or times out\n\n* Page: GitHub issue\n* URL: https://github.com/huginn/huginn/issues/2963\n* State: open\n* Author: BoKKeR\n* Created: 2021-03-14T16:42:01Z\n* Updated: 2021-03-16T19:09:24Z\n* Repository: huginn/huginn\n* Number: #2963\n* Comments: 1\nI know it might be controversial but I am trying to get a new GPU for a while. Wanted to setup a monitor for when the stock on amd.com changes. But the site just won\'t load. The WebsiteAgent gets stuck in\nFetching https://www.amd.com/de/direct-buy/se\nThe only way to stop it is by stopping the container. As I am running the service in a container I cant find any of the debug log locations. Can anyone else try if they are encountering the same error?\n{\n "debug": "true",\n "expected_update_period_in_days": "2",\n "url": "https://www.amd.com/de/direct-buy/se",\n "type": "html",' + ], + }, + { + "url": "https://github.com/mozilla-firefox/firefox-crb/blob/main/docs/bug-mgmt/guides/severity.md", + "title": "firefox-crb/docs/bug-mgmt/guides/severity.md at main · mozilla-firefox/firefox-crb · GitHub", + "excerpts": [ + "We use the severity field in Bugzilla to indicate the scope of a bug's effect on Firefox.\nThe field is display alongside the bug's priority.\n\nValues\n\nSeverity levels and their definitions are enumerated at https://wiki.mozilla.org/BMO/UserGuide/BugFields#bug_severity.\nBy default, new bugs have a severity of --.\n\nExamples of S1 bugs\n\n* WebExtensions disabled for all users\n* Web search not working from URL bar\n* Crashes with data loss\n\nExamples of S2 bugs\n\nBugs that could reasonably be expected to cause a Firefox user to switch browsers, either because the severity is bad enough, or the frequency of occurrence is high enough.\n* Bug 1640441 - Slack hangs indefinitely in a onResize loop" + ], + }, + { + "url": "https://github.com/n0-computer/iroh/issues/4604", + "title": "Endpoint::close hangs forever if the tokio runtime gets dropped while the connection is open · Issue #4604 · n0-computer/iroh · GitHub", + "excerpts": [ + "@alpergundogdu\nalpergundogdu\nopened on Oct 14, 2026\nIssue body actions\nDescribe the bug\nAfter opening a connection from inside a short-lived tokio runtime, and dropping the runtime while the connection is still open, Endpoint::close() never returns.\nIn my repro it hangs reliably; when I keep the runtime open, it closes instantly.\nLocal repro/test (has the control tests too): https://github.com/decdn/iroh-tests/blob/main/tests/bug_endpoint_close_after_runtime_drop.rs\n\n...\n\nIf spawning on the caller's runtime is intended, a note on Endpoint::connect saying the driver lives there would have saved us a lot of time. (Maybe it's an expectation/documentation issue?)\nIroh\nVersion: iroh 1.3.0, noq 1.3.0\nEndpoint configuration: bound to 127.0.0.1, no relays.\nPlatform(s)\nDesktop:\n\n...\n\nSorry @flub , that was unclear: the endpoint's runtime stays alive, and close() is called from it. Only a separate short-lived runtime that runs connect() gets dropped." + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1640441", + "title": "1640441 - Slack hangs indefinitely in a onResize loop", + "excerpts": [ + "Closed Bug 1640441 Opened 6 years ago Closed 6 years ago\n\nSlack hangs indefinitely in a on Resize loop\n\nCategories\n\n(Web Compatibility :: Site Reports, defect, P1)\n\nProduct:\nWeb Compatibility ▾\nComponent:\nSite Reports ▾\nType:\ndefect\nPriority:\nP1 Severity:\nS2\n\nTracking\n\n(Not tracked)\n\nStatus:\nRESOLVED FIXED\n\nPeople\n\n(Reporter: Harald, Assigned: emilio)\n\nReferences\n\n(Depends on 1 open bug)\n\nDetails\n\n(Keywords: enterprise, perf, webcompat:site-wait)\n\nAttachments\n\n(1 file, 1 obsolete file)\n\n|relevant unminified code.\n6 years ago Emilio Cobos Álvarez [:emilio]\n* 13.50 KB,\n\n...\n\n* Hover over the link\n* ... tab immediately becomes unresponsive ...\nThis may well be a bug in slack, but IMO it's still on us to try and fix what's going on. When it happens, it affects not only Slack but my entire Firefox session. This is a serious usability issue -- I'm now at the point of restarting Firefox several times a day because of this bug, if" + ], + }, + { + "url": "https://www.ibm.com/support/pages/cognos-business-intelligence-1022-fix-lists", + "title": "Cognos Business Intelligence 10.2.2 Fix Lists", + "excerpts": [ + 'IBM Support \n\nCognos Business Intelligence 10.2.2 Fix Lists\n\nRelease Notes\n\nAbstract\n\nA comprehensive list of defect corrections for major releases, refresh packs and fix packs of Cognos Business Intelligence 10.2.2.\nDetails of the APARs listed below can be accessed by clicking the link for the APAR number. If you have questions about a particular defect, please contact Customer Support.\n\n...\n\n* PI05143 | MULTIPLE VALUES IN CONTEXT FILTERS CAUSE A CROSSTAB REPORT TO SHOW DASH "-" INSTEAD OF CORRECT VALUES\n* PI05166 | QE-DEF-0359 The query contains a reference to at least one object that does not exist\n* PI05233 | Export deployments from 8.4.1 with export folder or package tasks in Content Administration contain unknown content steps\n\n...\n\n* PI08523 | Progressive Chart - Incorrect Colours when "Reverse Catagory Order" = True\n* PI08585 | Cognos hangs when starting against a Content Store from previous version (upgrade) and using SSL gateway UR' + ], + }, + { + "url": "https://josm.eu/ticket/8175", + "title": "\n #8175 ([PATCH] Cadastre grab query sometimes hangs completely)\n – JOSM\n\n ", + "excerpts": [ + 'Opened 14 years ago\nClosed 14 years ago\nLast modified 14 years ago\n\n#8175 closed defect ( fixed )\n\n[PATCH] Cadastre grab query sometimes hangs completely\n\n* Reported by: | Don-vip | Owned by: | pieren\n* Priority: | normal | Milestone: | \n* Component: | Plugin cadastre-fr | Version: | \n* Keywords: | | Cc: | \n\nDescription\n\nDuring cadastre grab, the HTTP connection with the WMS server sometimes completely hangs (the progress dialog is frozen and cancel button does not work).\nThe only solution is to restart JOSM to have a clean state and grab the cadastre again. We had a similar problem with history request ( #7640 ) when dealing with the API and solved it in [r5324](https://josm.eu/changeset/5324/josm "fix #7640 - History query will not cancel or timeout\n\n...\n\nby Don-vip , 14 years ago\n\n* Attachment: | 8175.patch \u200b added\n\ncomment:1 by Don-vip , 14 years ago' + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1749908", + "title": "1749908 - Infinite loop in HTTP3 hangs socket thread - Bugzilla", + "excerpts": [ + "Closed Bug 1749908 Opened 4 years ago Closed 4 years ago\n\nInfinite loop in HTTP3 hangs socket thread\n\nCategories\n\n(Core :: Networking, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking ▾\nPlatform:\nDesktop\nAll\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nVERIFIED DUPLICATE of bug 1749910\n\nPeople\n\n(Reporter: heftig, Unassigned)\n\nDetails\n\nBottom ↓\nTags ▾\nTimeline ▾\n| |Jan Alexander Steffens [:heftig]\n* Reporter | \n* ### Description\n•\n* 4 years ago\n* \nWhile browsing, pages suddenly stopped loading.\n\n...\n\nIt's not related to a specific version, we're getting reports ESR is even affected. Suspicion is around some long existing HTTP3 bug that's being triggered by an external service updating.\n\n...\n\nI agree it is not the root cause but it certainly caused a massive problem for many users of Firefox, the majority of whom are opted-in by default. If they had not been, they would not have been affected." + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1748414", + "title": "1748414 - Firefox 94 sometimes hangs on TLSv1.3 requests", + "excerpts": [ + "Closed Bug 1748414 Opened 4 years ago Closed 4 years ago\n\nFirefox 94 sometimes hangs on TLSv1 .3 requests\n\nCategories\n\n(Core :: Networking: HTTP, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking: HTTP ▾\nVersion:\nFirefox 94\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nRESOLVED DUPLICATE of bug 1740856\n\nPeople\n\n(Reporter: code, Unassigned)\n\nDetails\n\nBottom ↓\n\n...\n\nSteps to reproduce:\nIn the beginning of November, colleagues started to experience hangs of 1 to 1.5 minutes when working with our own web app. We use latest official releases for Windows, Ubuntu and FreeBSD. The problems exist for all plattforms and very likely started with Firefox 94.\nActual results:\nSometimes while interacting normally with our web app, a request hangs for a long time (up to 1.5 minutes; but usually succeeds). This typically happens on POST requests." + ], + }, +] +DECISION_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "parallel_ai/decision", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"}, + expected_provider_request={ + "mode": "fast", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "mode": "fast", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, + expected_litellm_response={ + "model": "decision", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.6442559258537284}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [ + {"value": "low", "probability": 0.1616964510265985}, + {"value": "high", "probability": 0.8383035489734014}, + ], + "confidence": 0.6766070979468028, + }, + { + "type": "score", + "name": "confidence", + "score": 0.25195637565890455, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.7480436243410955}, + {"value": 1, "label": "sure", "probability": 0.25195637565890455}, + ], + "confidence": 0.4960872486821909, + }, + ], + "usage": { + "input_tokens": 3166, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 3166, + }, + "mode": "fast", + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, +) +DECISION_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "parallel_ai/decision", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"}, + expected_provider_request={ + "mode": "fast", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "mode": "fast", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, + expected_litellm_response={ + "model": "decision", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "mode": "fast", + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py new file mode 100644 index 00000000000..47eb057323b --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.parallel_ai import DECISION_SYSTEMONE_TEST_CASE, DECISION_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [DECISION_TEST_CASE, DECISION_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_parallel_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 054e5eb1c65..2e9bf140f7f 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -354,7 +354,7 @@ class TestComplexityRouterInit: ) @pytest.mark.asyncio - async def test_jev_choice_without_confidence_uses_the_selected_probability(self, mock_router_instance): + async def test_jev_choice_without_confidence_routes_and_reports_no_confidence(self, mock_router_instance): client = _StaticJevClient( JevSystemOneResponse( answers={ @@ -379,11 +379,25 @@ class TestComplexityRouterInit: ) outcome = await router.aclassify("Explain this") + result = await router.async_pre_routing_hook( + model="test-router", + request_kwargs={}, + messages=[{"role": "user", "content": "Explain this"}], + ) assert outcome.tier == ComplexityTier.MEDIUM assert outcome.cause == "jev_classifier" assert outcome.jev_verdict is not None - assert outcome.jev_verdict.confidence == 0.9 + assert outcome.jev_verdict.confidence is None + assert outcome.signals == ( + "jev-classifier:MEDIUM", + "tier-probability:SIMPLE=0.100000", + "tier-probability:MEDIUM=0.900000", + ) + assert result is not None + assert result.routing_decision is not None + assert result.routing_decision["classifier_probabilities"] == {"SIMPLE": 0.1, "MEDIUM": 0.9} + assert "classifier_confidence" not in result.routing_decision @pytest.mark.asyncio async def test_jev_pre_routing_hook_exposes_routing_decision_provenance( diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index caeb0645fc9..8c11ea2d75e 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -45,6 +45,12 @@ describe("provider_info_helpers", () => { }); describe("getProviderLogoAndName", () => { + it("shows the Parallel AI logo for the parallel_ai slug", () => { + const { logo, displayName } = getProviderLogoAndName("parallel_ai"); + expect(displayName).toBe(Providers.ParallelAI); + expect(logo).toContain("parallel_ai"); + }); + it("should return empty logo and dash display name when providerValue is empty", () => { const result = getProviderLogoAndName(""); expect(result).toEqual({ logo: "", displayName: "-" }); @@ -230,7 +236,6 @@ describe("provider_info_helpers", () => { Providers.NLP_CLOUD, Providers.NSCALE, Providers.OVHCLOUD, - Providers.ParallelAI, Providers.PETALS, Providers.PG_VECTOR, Providers.PREDIBASE, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index beb9641e222..0fbf33a19b3 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -48,6 +48,7 @@ import ollamaLogo from "../../public/assets/logos/ollama.svg"; import openaiSmallLogo from "../../public/assets/logos/openai_small.svg"; import openrouterLogo from "../../public/assets/logos/openrouter.svg"; import oracleLogo from "../../public/assets/logos/oracle.svg"; +import parallelAiLogo from "../../public/assets/logos/parallel_ai.png"; import perplexityAiLogo from "../../public/assets/logos/perplexity-ai.svg"; import qwenLogo from "../../public/assets/logos/qwen.png"; import recraftLogo from "../../public/assets/logos/recraft.svg"; @@ -410,6 +411,7 @@ export const providerLogoMap: Partial> = { [Providers.OpenAI_Compatible]: openaiSmallLogo.src, [Providers.Openrouter]: openrouterLogo.src, [Providers.Oracle]: oracleLogo.src, + [Providers.ParallelAI]: parallelAiLogo.src, [Providers.Perplexity]: perplexityAiLogo.src, [Providers.Qwen_AI_Platform]: qwenLogo.src, [Providers.QwenCloud]: qwenLogo.src, From 56039b920779e4aeac4e1c7f468886ec4484dcde Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:21:06 -0700 Subject: [PATCH 7/8] fix(decisions): keep a token-priced row on token billing when it also sets a query price A plain input_cost_per_query bills flat only on a row with no token price, and input_cost_per_query_decisions always bills flat, so a custom Decisions row that already bills tokens costs the same as before this PR --- litellm/cost_calculator.py | 14 +++++---- .../litellm_core_utils/decisions_pricing.py | 9 ++++++ .../parallel_ai/decisions/transformation.py | 7 ----- .../test_decisions_pricing.py | 30 +++++++++++++++++++ 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 2b64e30ec6c..8c764c3d90d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -24,7 +24,11 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount -from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type +from litellm.litellm_core_utils.decisions_pricing import ( + decisions_pricing_fields, + decisions_request_price, + is_decisions_call_type, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -604,11 +608,9 @@ def cost_per_token( decisions_overrides: Final = decisions_pricing_fields( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) - request_price: Final = decisions_overrides.get( - "input_cost_per_query", decisions_model_info.get("input_cost_per_query") - ) - if isinstance(request_price, (int, float)): - return float(request_price), 0.0 + request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides) + if request_price is not None: + return request_price, 0.0 if decisions_overrides: return generic_cost_per_token( model=model, diff --git a/litellm/litellm_core_utils/decisions_pricing.py b/litellm/litellm_core_utils/decisions_pricing.py index 9199a8097cc..c6e38cd5bf1 100644 --- a/litellm/litellm_core_utils/decisions_pricing.py +++ b/litellm/litellm_core_utils/decisions_pricing.py @@ -27,6 +27,15 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]: } +def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None: + rates: Final = {**model_info, **overrides} + request_price: Final = rates.get("input_cost_per_query") + if not isinstance(request_price, (int, float)): + return None + bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token")) + return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price) + + def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]: """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields.""" plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)} diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py index 006d7499271..b7e4586e458 100644 --- a/litellm/llms/parallel_ai/decisions/transformation.py +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -1,10 +1,3 @@ -"""Parallel's decision model uses System One questions and the native fast mode. - -Both LiteLLM Decisions routes share this adapter. Citations describe the whole -request, so they stay on the response rather than being assigned to answers. -Contract: https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions -""" - from collections.abc import Mapping from typing import Final diff --git a/tests/unit/litellm_core_utils/test_decisions_pricing.py b/tests/unit/litellm_core_utils/test_decisions_pricing.py index b006850f624..720c973876a 100644 --- a/tests/unit/litellm_core_utils/test_decisions_pricing.py +++ b/tests/unit/litellm_core_utils/test_decisions_pricing.py @@ -77,6 +77,36 @@ def test_flat_decisions_rate_overrides_catalog_without_changing_chat( ) == pytest.approx((100 * 1e-07, 0.0)) +@pytest.mark.parametrize("call_type", ["adecisions", "asystemone"]) +@pytest.mark.parametrize( + ("token_rates", "expected_costs"), + [ + ({"input_cost_per_token": 1e-05, "output_cost_per_token": 0.0}, (100 * 1e-05, 0.0)), + ({"input_cost_per_token": 0.0, "output_cost_per_token": 1e-05}, (0.0, 10 * 1e-05)), + ], +) +def test_plain_query_price_leaves_a_token_priced_row_billing_tokens( + monkeypatch: pytest.MonkeyPatch, + call_type: str, + token_rates: dict[str, float], + expected_costs: tuple[float, float], +) -> None: + model: Final = "fireworks_ai/accounts/example/models/priced-both" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_query": 0.5, **token_rates, "litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + + assert cost_calculator.cost_per_token( + model=model, + custom_llm_provider="fireworks_ai", + prompt_tokens=100, + completion_tokens=10, + call_type=cast(CallTypesLiteral, call_type), + ) == pytest.approx(expected_costs) + + _LUNA_USAGE: Final = Usage( prompt_tokens=100, completion_tokens=10, From 6f332b8c808e03e93b034b0451436f7ded3f5563 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:49:51 -0700 Subject: [PATCH 8/8] fix(decisions): bill a deployment's query price and accept Parallel bases that end in a route A proxy request logs through an object built before the router picks the deployment, so a query price on litellm_params never reached Decisions billing. Parallel Search accepts a base ending in /v1/search and shares PARALLEL_AI_API_BASE with Decisions, so Decisions now accepts that base and one ending in /v1/decisions. --- litellm/decisions/call.py | 1 + .../parallel_ai/decisions/transformation.py | 3 + .../decisions/test_transformation.py | 59 ++++++++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index a48e3a2a89c..017f72b2a37 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -557,6 +557,7 @@ def prepare_call( litellm_params={ "litellm_call_id": kwargs.get("litellm_call_id"), "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + "input_cost_per_query": kwargs.get("input_cost_per_query"), }, custom_llm_provider=provider, ) diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py index b7e4586e458..2a6519a99ff 100644 --- a/litellm/llms/parallel_ai/decisions/transformation.py +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -41,6 +41,9 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig): ) return resolved_base, self.resolve_api_key(None) + def get_complete_url(self, api_base: str, model: str) -> str: + return super().get_complete_url(api_base.rstrip("/").removesuffix("/v1/search").removesuffix(self.path), model) + def _decisions_url(self, api_base: str | None) -> httpx.URL: return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision")) diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py index 5bc3c96fe1f..1fae28a329b 100644 --- a/tests/unit/llms/parallel_ai/decisions/test_transformation.py +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -1,6 +1,7 @@ import json from collections.abc import Mapping from dataclasses import replace +from datetime import datetime from typing import Final import httpx @@ -23,7 +24,7 @@ from litellm.types.decisions import ( UnsupportedDecisionsRequest, ) from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, Rules, function_setup QUESTIONS: Final = { "available": {"type": "noul", "instructions": "Is this product available?"}, @@ -98,6 +99,12 @@ def test_provider_routes_systemone_questions_to_parallel_without_a_model_field() assert config.get_complete_url("https://gateway.example/v1/", "decision") == "https://gateway.example/v1/decisions" +@pytest.mark.parametrize("suffix", ["", "/", "/v1", "/v1/", "/v1/decisions", "/v1/decisions/", "/v1/search", "/v1/search/"]) +def test_base_carrying_a_parallel_route_reaches_the_decisions_endpoint(suffix: str) -> None: + complete_url: Final = ParallelAIDecisionsConfig().get_complete_url(f"https://api.parallel.ai{suffix}", "decision") + assert complete_url == "https://api.parallel.ai/v1/decisions" + + def test_auth_replaces_conflicting_headers_without_forwarding_bearer_credentials() -> None: headers: Final = ParallelAIDecisionsConfig().validate_environment( {"Authorization": "Bearer old", "X-API-Key": "old", "CONTENT-TYPE": "text/plain", "X-Trace": "test"}, @@ -226,6 +233,38 @@ def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: s assert cost == (rate, 0.0) +@pytest.mark.asyncio +async def test_deployment_query_price_replaces_the_catalog_price( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) + price: Final = 25 * litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "webjev", + "litellm_params": { + "model": "parallel_ai/decision", + "api_key": "test-key", + "input_cost_per_query": price, + }, + } + ] + ) + fields: Final = { + "model": "webjev", + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + "litellm_call_id": "query-price-call", + } + logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields) + response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj) + assert response._hidden_params["response_cost"] == pytest.approx(price) + await GLOBAL_LOGGING_WORKER.clear_queue() + + @pytest.mark.parametrize( "payload", [{"answers": {}}, {"mode": "fast", "answers": {"available": {"type": "noul", "noul": "invalid"}}}] ) @@ -279,6 +318,9 @@ async def test_untrusted_base_cannot_receive_environment_key( "https://api.parallel.ai:8443", "https://attacker.example/api.parallel.ai", "https://api.parallel.ai/other-tenant", + "https://api.parallel.ai/other-tenant/v1/search", + "https://attacker.example/v1/search", + "https://attacker.example/v1/decisions", ], ) def test_environment_key_rejects_untrusted_destination_variants(api_base: str, monkeypatch: pytest.MonkeyPatch) -> None: @@ -288,7 +330,16 @@ def test_environment_key_rejects_untrusted_destination_variants(api_base: str, m ParallelAIDecisionsConfig().resolve_connection(api_base, None) -@pytest.mark.parametrize("api_base", [None, "https://api.parallel.ai", "https://API.PARALLEL.AI:443/v1/"]) +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.parallel.ai", + "https://API.PARALLEL.AI:443/v1/", + "https://api.parallel.ai/v1/decisions", + "https://api.parallel.ai/v1/search", + ], +) def test_default_destination_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") @@ -297,7 +348,9 @@ def test_default_destination_retains_environment_key(api_base: str | None, monke assert key == "server-secret" -@pytest.mark.parametrize("api_base", [None, "https://gateway.example/parallel/v1/"]) +@pytest.mark.parametrize( + "api_base", [None, "https://gateway.example/parallel/v1/", "https://gateway.example/parallel/v1/search"] +) def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://gateway.example/parallel") monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret")