fix(decisions): keep per-query billing and optional confidence on Parallel only

A query price on any other Decisions provider moved that provider off token
billing, and a choice or score answer with no confidence became a 200 with a
null for every provider. Both now apply to Parallel alone, so other providers
bill and fail as they do on main.

A Parallel deployment with explicit zero token rates and a query price read as
free, so every budget check was skipped while it billed per call. It now counts
as priced.

The dashboard key for Parallel is now PARALLEL_AI so a served bundle that
predates the provider still sends parallel_ai as the provider slug, and the
README table lists the provider.
This commit is contained in:
mateo-berri 2026-10-31 17:48:31 -07:00
parent 6f332b8c80
commit 0a0095d616
15 changed files with 197 additions and 22 deletions

View file

@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) | | | | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -13,6 +13,7 @@ from typing_extensions import assert_never
import litellm
from litellm.cost_calculator import cost_map_key_prices_anything
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -557,7 +558,9 @@ def prepare_call(
litellm_params={
"litellm_call_id": kwargs.get("litellm_call_id"),
"api_base": provider_config.get_complete_url(resolved_api_base, canonical_model),
"input_cost_per_query": kwargs.get("input_cost_per_query"),
"input_cost_per_query": (
kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None
),
},
custom_llm_provider=provider,
)

View file

@ -12,6 +12,7 @@ from typing import Final
DECISIONS_PRICING_SUFFIX: Final = "_decisions"
DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"})
QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"})
def is_decisions_call_type(call_type: str) -> bool:
@ -28,6 +29,8 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]:
def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None:
if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS:
return None
rates: Final = {**model_info, **overrides}
request_price: Final = rates.get("input_cost_per_query")
if not isinstance(request_price, (int, float)):

View file

@ -436,6 +436,10 @@ class DecisionsResponseValidationError(BaseLLMException):
pass
def _lacks_confidence(answer: DecisionsIRAnswer) -> bool:
return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
return systemone_response_to_ir(
_SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request
@ -524,6 +528,7 @@ class BaseDecisionsConfig(ABC):
image_media_types: frozenset[str] | None = None
inlines_remote_images: bool = False
openai_evidence_fields: frozenset[str] = frozenset()
confidence_optional: bool = False
health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
{"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
)
@ -660,13 +665,20 @@ class BaseDecisionsConfig(ABC):
) -> DecisionsIRResponse:
try:
payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content)
return self.parse_response(payload, request)
response: Final = self.parse_response(payload, request)
except ValidationError as error:
raise DecisionsResponseValidationError(
status_code=500,
message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
f"{error.title} {_validation_summary(error)}",
) from error
if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers):
raise DecisionsResponseValidationError(
status_code=500,
message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
"a choice or score answer has no confidence",
)
return response
def get_error_class(
self,

View file

@ -23,6 +23,7 @@ class ParallelAIDecisionsConfig(BaseDecisionsConfig):
path = "/v1/decisions"
api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY")
api_base_env = ("PARALLEL_AI_API_BASE",)
confidence_optional = True
openai_evidence_fields = frozenset({"citations", "decision_id", "mode"})
def get_default_api_base(self) -> str:

View file

@ -43,7 +43,9 @@ from litellm.constants import (
REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
TAG_REGISTRY_MAX_SIZE,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, decisions_request_price
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.tool_fields import DropPolicy
@ -727,7 +729,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -
def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
"""Whether any of the deployments carries an explicit positive per-token price.
"""Whether any of the deployments carries an explicit positive per-token or per-query price.
The group's price is read from the deployments named after the routed group and from
that group's own alias target, while an alias chain is served from the deployments named
@ -738,14 +740,21 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
if model_id is None:
continue
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost(
raw_entry.get("output_cost_per_token")
raw_entry = as_str_mapping(litellm.model_cost.get(model_id)) or _EMPTY_COST_ENTRY
if (
_is_positive_cost(raw_entry.get("input_cost_per_token"))
or _is_positive_cost(raw_entry.get("output_cost_per_token"))
or _is_positive_cost(_deployment_query_price(deployment["litellm_params"], raw_entry))
):
return True
return False
def _deployment_query_price(litellm_params: Mapping[str, object], cost_entry: Mapping[str, object]) -> float | None:
provider: Final = litellm_params.get("custom_llm_provider") or str(litellm_params.get("model")).partition("/")[0]
return decisions_request_price({**cost_entry, "litellm_provider": provider}, decisions_pricing_fields(cost_entry))
_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})

View file

@ -2614,7 +2614,7 @@
"default_model_placeholder": "nace/drex-latest"
},
{
"provider": "ParallelAI",
"provider": "PARALLEL_AI",
"provider_display_name": "Parallel AI",
"litellm_provider": "parallel_ai",
"credential_fields": [

View file

@ -66,15 +66,30 @@ def test_apply_decisions_pricing_leaves_rows_without_decisions_fields_unchanged(
def test_flat_decisions_rate_overrides_catalog_without_changing_chat(
monkeypatch: pytest.MonkeyPatch, rate: float
) -> None:
model: Final = "parallel_ai/decision"
monkeypatch.setitem(
litellm.model_cost,
"gpt-6-luna",
{**litellm.model_cost["gpt-6-luna"], **_ROW, "input_cost_per_query_decisions": rate},
model,
{**_ROW, "litellm_provider": "parallel_ai", "input_cost_per_query_decisions": rate},
)
assert cost_calculator.cost_per_token(model="gpt-6-luna", prompt_tokens=100, call_type="decisions") == (rate, 0.0)
assert cost_calculator.cost_per_token(
model="gpt-6-luna", prompt_tokens=100, call_type="completion"
) == pytest.approx((100 * 1e-07, 0.0))
assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (rate, 0.0)
assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="completion") == pytest.approx(
(100 * 1e-07, 0.0)
)
@pytest.mark.parametrize("query_field", ["input_cost_per_query", "input_cost_per_query_decisions"])
def test_query_price_on_a_provider_that_bills_tokens_stays_unbilled(
monkeypatch: pytest.MonkeyPatch, query_field: str
) -> None:
model: Final = "fireworks_ai/accounts/example/models/query-priced"
monkeypatch.setitem(
litellm.model_cost,
model,
{query_field: 0.5, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
| {"litellm_provider": "fireworks_ai", "mode": "evaluation"},
)
assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (0.0, 0.0)
@pytest.mark.parametrize("call_type", ["adecisions", "asystemone"])

View file

@ -8,6 +8,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
DecisionsResponseValidationError,
ir_to_systemone_request,
ir_to_systemone_response,
parse_systemone_response,
@ -16,6 +17,7 @@ from litellm.llms.base_llm.decisions.transformation import (
from litellm.llms.openai.decisions.transformation import openai_request_to_ir
from litellm.types.decisions import (
MAX_DECISION_QUESTIONS,
DecisionsIRChoiceAnswer,
DecisionsIRRefusal,
DecisionsIRRequest,
DecisionsIRScoreAnswer,
@ -294,6 +296,49 @@ def test_a_box_answer_without_confidence_reaches_the_caller_unchanged() -> None:
assert response.model_dump(mode="json") == payload
@pytest.mark.parametrize(
("question", "answer"),
[
(
{"type": "choice", "criteria": {"yes": "Yes", "no": "No"}},
{"type": "choice", "choice": "yes", "probabilities": {"yes": 0.9, "no": 0.1}},
),
(
{"type": "score", "criteria": ["low", "high"]},
{"type": "score", "score": 0.5, "legend": {"0": "low", "1": "high"}, "probabilities": {"0": 0.5, "1": 0.5}},
),
],
)
@pytest.mark.parametrize("confidence", ({}, {"confidence": None}))
@pytest.mark.parametrize(
("provider", "optional"), (("nace", False), ("fireworks_ai", False), ("parallel_ai", True))
)
def test_only_a_provider_without_confidence_may_omit_it(
question: Mapping[str, object],
answer: Mapping[str, object],
confidence: Mapping[str, object],
provider: str,
optional: bool,
) -> None:
config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders(provider))
assert config is not None
request: Final = systemone_request_to_ir(
DecisionsRequestBody.model_validate({"state": {"text": "Example"}, "questions": {"asked": question}})
)
upstream: Final = httpx.Response(
200, json={"mode": "fast", "model": "decision", "answers": {"asked": {**answer, **confidence}}}
)
if optional:
parsed: Final = config.transform_decisions_response("decision", provider, upstream, request)
assert isinstance(parsed.answers[0], (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer))
assert parsed.answers[0].confidence is None
return
with pytest.raises(DecisionsResponseValidationError) as raised:
config.transform_decisions_response("decision", provider, upstream, request)
assert raised.value.status_code == 500
@pytest.mark.parametrize("provider", ("nace", "openrouter", "fireworks_ai", "typesafe"))
@pytest.mark.parametrize("legend", (["low", "high"], {"0": "low", "1": "high"}))
def test_score_legends_preserve_labels_probabilities_and_metadata(provider: str, legend: object) -> None:

View file

@ -2,10 +2,15 @@ from collections.abc import Mapping
from dataclasses import replace
from typing import Final
import httpx
import pytest
from pydantic import TypeAdapter
from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir
from litellm.llms.base_llm.decisions.transformation import (
DecisionsResponseValidationError,
ir_to_systemone_response,
systemone_request_to_ir,
)
from litellm.llms.openai.decisions.transformation import (
OpenAIDecisionsConfig,
ir_to_openai_request,
@ -107,6 +112,22 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None:
assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE
@pytest.mark.parametrize("answer_type", ["choice", "score"])
def test_a_choice_or_score_answer_without_confidence_stays_a_provider_error(answer_type: str) -> None:
answers: Final = [
{key: value for key, value in answer.items() if answer["type"] != answer_type or key != "confidence"}
for answer in _OPENAI_RESPONSE["answers"]
]
with pytest.raises(DecisionsResponseValidationError) as raised:
OpenAIDecisionsConfig().transform_decisions_response(
"gpt-6-luna",
"openai",
httpx.Response(200, json={**_OPENAI_RESPONSE, "answers": answers}),
_openai_ir(_OPENAI_REQUEST),
)
assert raised.value.status_code == 500
def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None:
ir: Final = _openai_ir(_OPENAI_REQUEST)
parsed: Final = replace(

View file

@ -265,6 +265,48 @@ async def test_deployment_query_price_replaces_the_catalog_price(
await GLOBAL_LOGGING_WORKER.clear_queue()
@pytest.mark.asyncio
async def test_query_price_on_another_provider_leaves_its_token_billing_alone(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
input_rate: Final = 1e-06
tokens: Final = 1000
model: Final = "fireworks_ai/accounts/example/models/token-billed"
monkeypatch.setitem(
litellm.model_cost,
model,
{"input_cost_per_token": input_rate, "output_cost_per_token": 0.0}
| {"litellm_provider": "fireworks_ai", "mode": "evaluation"},
)
respx_mock.post("https://api.fireworks.ai/inference/v1/systemone").respond(
json={
"model": "token-billed",
"answers": {"available": {"type": "noul", "noul": 0.9}},
"usage": {"input_tokens": tokens, "output_tokens": 0},
}
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "token-billed",
"litellm_params": {"model": model, "api_key": "test-key", "input_cost_per_query": 0.025},
}
]
)
fields: Final = {
"model": "token-billed",
"input": "Example",
"questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}],
"litellm_call_id": "token-billed-call",
}
logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields)
response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj)
assert response._hidden_params["response_cost"] == pytest.approx(tokens * input_rate)
await GLOBAL_LOGGING_WORKER.clear_queue()
@pytest.mark.parametrize(
"payload", [{"answers": {}}, {"mode": "fast", "answers": {"available": {"type": "noul", "noul": "invalid"}}}]
)

View file

@ -486,7 +486,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
(
("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE),
("Nace", "Nace", LlmProviders.NACE),
("ParallelAI", "Parallel AI", LlmProviders.PARALLEL_AI),
("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI),
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
),
)

View file

@ -32,6 +32,29 @@ from litellm.router import Router
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@pytest.mark.parametrize(
("model", "expected_free"),
[("parallel_ai/decision", False), ("fireworks_ai/accounts/fireworks/routers/rune", True)],
)
def test_query_price_keeps_budgets_only_where_it_is_billed(model: str, expected_free: bool) -> None:
router = Router(
model_list=[
{
"model_name": "query-priced",
"litellm_params": {
"model": model,
"api_key": "test-key",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"input_cost_per_query": 0.025,
},
}
]
)
assert is_model_cost_zero("query-priced", router) is expected_free
@pytest.fixture
def mock_router_with_zero_cost_model():
"""Create a mock router with a zero-cost model."""

View file

@ -47,7 +47,7 @@ describe("provider_info_helpers", () => {
describe("getProviderLogoAndName", () => {
it("shows the Parallel AI logo for the parallel_ai slug", () => {
const { logo, displayName } = getProviderLogoAndName("parallel_ai");
expect(displayName).toBe(Providers.ParallelAI);
expect(displayName).toBe(Providers.PARALLEL_AI);
expect(logo).toContain("parallel_ai");
});
@ -560,9 +560,9 @@ describe("provider_info_helpers", () => {
"parallel_ai/decision": { litellm_provider: "parallel_ai", mode: "evaluation" },
};
expect(getProviderModels("ParallelAI", modelMap)).toEqual(["parallel_ai/decision"]);
expect(getProviderModels("PARALLEL_AI", modelMap)).toEqual(["parallel_ai/decision"]);
expect(getProviderLogoAndName("parallel_ai").displayName).toBe("Parallel AI");
expect(getPlaceholder("ParallelAI")).toBe("parallel_ai/decision");
expect(getPlaceholder("PARALLEL_AI")).toBe("parallel_ai/decision");
});
it("lists Nace decision models and resolves the provider name and model placeholder", () => {

View file

@ -187,7 +187,7 @@ export enum Providers {
Triton = "Triton",
TypeSafe = "TypeSafe",
Nace = "Nace",
ParallelAI = "Parallel AI",
PARALLEL_AI = "Parallel AI",
V0 = "V0",
VERCEL_AI_GATEWAY = "Vercel Ai Gateway",
Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)",
@ -319,7 +319,7 @@ export const provider_map: Record<string, string> = {
Triton: "triton",
TypeSafe: "typesafe",
Nace: "nace",
ParallelAI: "parallel_ai",
PARALLEL_AI: "parallel_ai",
V0: "v0",
VERCEL_AI_GATEWAY: "vercel_ai_gateway",
Vertex_AI: "vertex_ai",
@ -411,7 +411,7 @@ export const providerLogoMap: Partial<Record<Providers, string>> = {
[Providers.OpenAI_Compatible]: openaiSmallLogo.src,
[Providers.Openrouter]: openrouterLogo.src,
[Providers.Oracle]: oracleLogo.src,
[Providers.ParallelAI]: parallelAiLogo.src,
[Providers.PARALLEL_AI]: parallelAiLogo.src,
[Providers.Perplexity]: perplexityAiLogo.src,
[Providers.Qwen_AI_Platform]: qwenLogo.src,
[Providers.QwenCloud]: qwenLogo.src,
@ -504,7 +504,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
[Providers.Tencent]: "tencent/deepseek-v4-pro",
[Providers.TypeSafe]: "typesafe/jev-latest",
[Providers.Nace]: "nace/drex-latest",
[Providers.ParallelAI]: "parallel_ai/decision",
[Providers.PARALLEL_AI]: "parallel_ai/decision",
[Providers.StreamLake]: "streamlake/<model-or-endpoint>",
[Providers.Vertex_AI]: "gemini-pro",
[Providers.VolcEngine]: "volcengine/<any-model-on-volcengine>",