From 19c62b8b35dd3dcc4725a1c7395eedd1fb3fe961 Mon Sep 17 00:00:00 2001 From: James Liounis Date: Fri, 21 Aug 2026 15:38:00 -0400 Subject: [PATCH 1/5] fix(cost): charge input_cost_per_request in the generic cost path The cost map has carried an input_cost_per_request key for a long time, but no code on the generic path reads it: generic_cost_per_token never adds it, and cost_per_token's generic fallback treats a model priced only per request as having no pricing at all and returns zero. Any provider on the generic path with flat per-request pricing therefore tracks $0 spend. This adds the key to ModelInfo, charges it in generic_cost_per_token on top of any token pricing, and admits it to the generic fallback's guard. Providers with a dedicated cost_per_token branch (perplexity's online models, for example) do not route through this path and are unaffected. --- litellm/cost_calculator.py | 1 + .../litellm_core_utils/llm_cost_calc/utils.py | 5 + litellm/types/utils.py | 2 + litellm/utils.py | 1 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 111 ++++++++++++++++++ 5 files changed, 120 insertions(+) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8f7cd09d364..7b46f8059f4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -653,6 +653,7 @@ def cost_per_token( if ( (model_info.get("input_cost_per_token") or 0.0) > 0 or (model_info.get("output_cost_per_token") or 0.0) > 0 + or (model_info.get("input_cost_per_request") or 0.0) > 0 or model_info.get("tiered_pricing") is not None ): return generic_cost_per_token( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 0a52e1d283e..099b3a40e5a 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -918,6 +918,11 @@ def generic_cost_per_token( service_tier=service_tier, ) + ## FLAT PER-REQUEST COST + input_cost_per_request: Final = model_info.get("input_cost_per_request") + if input_cost_per_request: + prompt_cost += input_cost_per_request + ## CALCULATE OUTPUT COST text_tokens = 0 audio_tokens = 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8a71d209618..80f56904367 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -233,6 +233,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x input input_cost_per_character_above_128k_tokens: float | None # only for vertex ai models input_cost_per_query: float | None # only for rerank models + input_cost_per_request: ReadOnly[float | None] # flat per-request pricing input_cost_per_image: float | None # only for vertex ai models input_cost_per_image_token: float | None # for gpt-image-1 and similar models input_cost_per_video_token: float | None # for gemini omni models with video input @@ -3323,6 +3324,7 @@ class MirroredPricingParams(BaseModel): class CustomPricingLiteLLMParams(MirroredPricingParams): ## CUSTOM PRICING ## input_cost_per_second: float | None = None + input_cost_per_request: float | None = None output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None input_cost_per_pixel: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index e5ce7157e77..92f7fde868a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5673,6 +5673,7 @@ def _get_model_info_helper( ), input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None), input_cost_per_query=_model_info.get("input_cost_per_query", None), + input_cost_per_request=_model_info.get("input_cost_per_request", None), input_cost_per_second=_model_info.get("input_cost_per_second", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index f66056a54e2..a167a3cdf2f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -3604,3 +3604,114 @@ def test_generic_cost_per_token_grok_46_long_context(_local_model_cost_map): ) assert prompt_cost == pytest.approx(200_000 * 4e-06 + 50_000 * 1e-06) assert completion_cost == pytest.approx(1_000 * 1.2e-05) + + +@pytest.fixture +def flat_per_request_model(): + """Register a synthetic model whose only price is a flat per-request rate.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + + model = "openai/flat-rate-test-model" + prev = litellm.model_cost.get(model) + litellm.model_cost[model] = { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openai", + "mode": "chat", + } + _invalidate_model_cost_lowercase_map() + try: + yield model + finally: + if prev is None: + litellm.model_cost.pop(model, None) + else: + litellm.model_cost[model] = prev + _invalidate_model_cost_lowercase_map() + + +def test_flat_per_request_cost_applies_once(flat_per_request_model): + """A flat per-request price bills the same regardless of token counts.""" + from litellm.types.utils import Usage + + small_usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + large_usage = Usage(prompt_tokens=10_000, completion_tokens=5_000, total_tokens=15_000) + + small_prompt_cost, small_completion_cost = generic_cost_per_token( + model=flat_per_request_model, + usage=small_usage, + custom_llm_provider="openai", + ) + large_prompt_cost, large_completion_cost = generic_cost_per_token( + model=flat_per_request_model, + usage=large_usage, + custom_llm_provider="openai", + ) + + assert small_prompt_cost == pytest.approx(0.05) + assert small_completion_cost == 0.0 + assert large_prompt_cost == pytest.approx(0.05) + assert large_completion_cost == 0.0 + + +def test_flat_per_request_cost_is_added_to_token_cost(): + """The flat rate stacks on top of token pricing rather than replacing it.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + from litellm.types.utils import Usage + + model = "openai/flat-plus-tokens-test-model" + litellm.model_cost[model] = { + "input_cost_per_request": 0.05, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 2e-05, + "litellm_provider": "openai", + "mode": "chat", + } + _invalidate_model_cost_lowercase_map() + try: + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="openai", + ) + assert prompt_cost == pytest.approx(0.05 + 100 * 1e-05) + assert completion_cost == pytest.approx(50 * 2e-05) + finally: + litellm.model_cost.pop(model, None) + _invalidate_model_cost_lowercase_map() + + +def test_flat_per_request_cost_routes_through_cost_per_token(): + """The generic cost_per_token dispatch must not skip a model whose only + pricing is a flat per-request rate. + + Uses a provider with no dedicated cost_per_token branch, so the model + genuinely reaches the generic fallback whose guard is under test. + """ + from litellm.cost_calculator import cost_per_token + from litellm.types.utils import Usage + from litellm.utils import _invalidate_model_cost_lowercase_map + + model = "groq/flat-rate-only-test-model" + litellm.model_cost[model] = { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "groq", + "mode": "chat", + } + _invalidate_model_cost_lowercase_map() + try: + usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + prompt_cost, completion_cost = cost_per_token( + model=model, + custom_llm_provider="groq", + usage_object=usage, + ) + assert prompt_cost == pytest.approx(0.05) + assert completion_cost == 0.0 + finally: + litellm.model_cost.pop(model, None) + _invalidate_model_cost_lowercase_map() From 66525bee2b702edc62ab4891ee075d1a87aa12fe Mon Sep 17 00:00:00 2001 From: James Liounis Date: Fri, 21 Aug 2026 16:44:31 -0400 Subject: [PATCH 2/5] chore(ui): regenerate dashboard API types for input_cost_per_request CustomPricingLiteLLMParams gained the field, so the proxy OpenAPI spec and the generated dashboard types move with it. --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index cf55dc69e86..f823577c336 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27521,6 +27521,8 @@ export interface components { input_cost_per_pixel?: number | null; /** Input Cost Per Query */ input_cost_per_query?: number | null; + /** Input Cost Per Request */ + input_cost_per_request?: number | null; /** Input Cost Per Second */ input_cost_per_second?: number | null; /** Input Cost Per Token */ @@ -36730,6 +36732,8 @@ export interface components { input_cost_per_pixel?: number | null; /** Input Cost Per Query */ input_cost_per_query?: number | null; + /** Input Cost Per Request */ + input_cost_per_request?: number | null; /** Input Cost Per Second */ input_cost_per_second?: number | null; /** Input Cost Per Token */ From 1ed4ae981aa0cfbec77924872e18c00d5f930505 Mon Sep 17 00:00:00 2001 From: James Liounis Date: Fri, 21 Aug 2026 15:51:17 -0400 Subject: [PATCH 3/5] feat(parallel_ai): add Parallel AI as an LLM provider (responses, chat, messages) Parallel AI is registered as a search-only provider today, so its OpenAI Responses-compatible endpoint is unreachable through the gateway: routing parallel_ai/parallel to /v1/responses, /v1/chat/completions or /v1/messages fails with 'LLM Provider NOT provided'. Registers parallel_ai as an LLM provider with a native Responses API config. completion() and /v1/messages are served through the existing responses bridge. Parallel exposes one model whose performance tier is selected by reasoning.effort, so cost tracking resolves the priced model from the effective effort, and parallel-low / -medium / -high aliases pin a tier and bill at its own published rate. A server-configured key is never forwarded to a caller-supplied api_base; an explicit api_key is required when overriding the base, matching the trust model the search adapter already uses. --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 3 + litellm/cost_calculator.py | 32 +- .../get_llm_provider_logic.py | 15 + litellm/llms/parallel_ai/common_utils.py | 52 +++ .../llms/parallel_ai/responses/__init__.py | 7 + .../parallel_ai/responses/cost_calculator.py | 41 ++ .../parallel_ai/responses/transformation.py | 120 ++++++ ...odel_prices_and_context_window_backup.json | 36 ++ litellm/types/utils.py | 1 + litellm/utils.py | 7 + model_prices_and_context_window.json | 36 ++ provider_endpoints_support.json | 6 +- ...st_parallel_ai_responses_transformation.py | 401 ++++++++++++++++++ .../llms/parallel_ai/test_common_utils.py | 123 ++++++ .../test_parallel_ai_responses_gateway.py | 138 ++++++ 17 files changed, 1022 insertions(+), 4 deletions(-) create mode 100644 litellm/llms/parallel_ai/common_utils.py create mode 100644 litellm/llms/parallel_ai/responses/__init__.py create mode 100644 litellm/llms/parallel_ai/responses/cost_calculator.py create mode 100644 litellm/llms/parallel_ai/responses/transformation.py create mode 100644 tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py create mode 100644 tests/test_litellm/llms/parallel_ai/test_common_utils.py create mode 100644 tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py diff --git a/litellm/__init__.py b/litellm/__init__.py index e95b553c5d4..a174f752837 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1786,6 +1786,9 @@ if TYPE_CHECKING: from .llms.manus.responses.transformation import ( ManusResponsesAPIConfig as ManusResponsesAPIConfig, ) + from .llms.parallel_ai.responses.transformation import ( + ParallelAIResponsesConfig as ParallelAIResponsesConfig, + ) from .llms.perplexity.responses.transformation import ( PerplexityResponsesConfig as PerplexityResponsesConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 89c72acc06d..9f082ecac22 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -238,6 +238,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMResponsesAPIConfig", "VolcEngineResponsesAPIConfig", "PerplexityResponsesConfig", + "ParallelAIResponsesConfig", "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", "BedrockMantleResponsesAPIConfig", @@ -961,6 +962,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.perplexity.responses.transformation", "PerplexityResponsesConfig", ), + "ParallelAIResponsesConfig": ( + ".llms.parallel_ai.responses.transformation", + "ParallelAIResponsesConfig", + ), "DatabricksResponsesAPIConfig": ( ".llms.databricks.responses.transformation", "DatabricksResponsesAPIConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 11904d5e9c2..a7b52cee1d1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -562,6 +562,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "ollama_chat", "deepinfra", "perplexity", + "parallel_ai", "mistral", "groq", "gigachat", @@ -741,6 +742,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES: Final = { openai_compatible_endpoints: Final[list] = [ "api.perplexity.ai", + "api.parallel.ai", "api.endpoints.anyscale.com/v1", "api.deepinfra.com/v1/openai", "api.mistral.ai/v1", @@ -801,6 +803,7 @@ openai_compatible_providers: Final[list] = [ "tencent", "deepinfra", "perplexity", + "parallel_ai", "xinference", "xai", "zai", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7b46f8059f4..c9527a40ded 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2,7 +2,9 @@ ## File for 'response_cost' calculation in Logging import logging import time +from collections.abc import Mapping from functools import lru_cache +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast from httpx import Response @@ -179,6 +181,7 @@ _SEARCH_CALL_TYPES: Final = frozenset( ) _AREALTIME_CALL_TYPE: Final = CallTypes.arealtime.value +EMPTY_OPTIONAL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _MCP_CALL_TYPE: Final = CallTypes.call_mcp_tool.value @@ -872,6 +875,20 @@ def _normalize_service_tier(service_tier: object) -> str | None: return service_tier +def _parallel_ai_response_pricing_model(model: str, optional_params: Mapping[str, object] | None) -> str | None: + from litellm.llms.parallel_ai.responses.cost_calculator import ( + is_parallel_ai_response_model, + parallel_ai_response_pricing_model, + ) + + if not is_parallel_ai_response_model(model): + return None + return parallel_ai_response_pricing_model( + model=model, + optional_params=optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS, + ) + + def _extract_service_tier(source: object) -> str | None: """Read a raw ``service_tier`` off a response body or usage object, dict or pydantic model alike.""" if isinstance(source, BaseModel): @@ -1228,12 +1245,25 @@ def completion_cost( service_tier = _normalize_service_tier(service_tier) + provider_for_cost: Final = _get_provider_for_cost_calc( + model=model, + custom_llm_provider=custom_llm_provider, + ) + pricing_base_model: Final = ( + _parallel_ai_response_pricing_model( + model=model or "parallel", + optional_params=optional_params, + ) + if base_model is None and custom_pricing is not True and provider_for_cost == LlmProviders.PARALLEL_AI.value + else base_model + ) + selected_model: Final = _select_model_name_for_cost_calc( model=model, completion_response=completion_response, custom_llm_provider=custom_llm_provider, custom_pricing=custom_pricing, - base_model=base_model, + base_model=pricing_base_model, router_model_id=router_model_id, ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index e674fc37673..adcee7014ad 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -233,6 +233,14 @@ def get_llm_provider( if endpoint == "api.perplexity.ai": custom_llm_provider = "perplexity" dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY") + elif endpoint == "api.parallel.ai": + from litellm.llms.parallel_ai.common_utils import ( + resolve_parallel_ai_credentials, + ) + + custom_llm_provider = "parallel_ai" # rebind-ok: function returns custom_llm_provider + resolved = resolve_parallel_ai_credentials(api_base=api_base, api_key=api_key) + api_base, dynamic_api_key = resolved # rebind-ok: function returns api_base elif endpoint == "api.endpoints.anyscale.com/v1": custom_llm_provider = "anyscale" dynamic_api_key = get_secret_str("ANYSCALE_API_KEY") @@ -550,6 +558,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "parallel_ai": + from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials + + # parallel_ai serves the OpenAI Responses-compatible API at https://api.parallel.ai/v1/responses + api_base, dynamic_api_key = resolve_parallel_ai_credentials( # rebind-ok: function returns api_base + api_base=api_base, api_key=api_key + ) elif custom_llm_provider == "aiohttp_openai": return model, "aiohttp_openai", api_key, api_base elif custom_llm_provider == "anyscale": diff --git a/litellm/llms/parallel_ai/common_utils.py b/litellm/llms/parallel_ai/common_utils.py new file mode 100644 index 00000000000..56e213c76f0 --- /dev/null +++ b/litellm/llms/parallel_ai/common_utils.py @@ -0,0 +1,52 @@ +""" +Shared credential and endpoint resolution for the Parallel AI provider. +""" + +from typing import Final +from urllib.parse import urlsplit + +from litellm.secret_managers.main import get_secret_str + +PARALLEL_AI_API_BASE: Final = "https://api.parallel.ai" + + +def _origin(url: str) -> tuple[str, str]: + """Scheme and host of a base URL; a scheme-less base is read as https.""" + normalized: Final = url if "://" in url else f"https://{url}" + split: Final = urlsplit(normalized) + return split.scheme.lower(), split.netloc.lower() + + +def _is_trusted_api_base(caller_api_base: str, env_api_base: str | None) -> bool: + """Whether a caller-supplied base names an origin the server key may be sent to. + + Compares scheme as well as host: matching the host alone would accept + ``http://api.parallel.ai`` and put the server key on the wire in plaintext. + """ + trusted: Final = frozenset(_origin(base) for base in (PARALLEL_AI_API_BASE, env_api_base) if base) + scheme, host = _origin(caller_api_base) + return bool(host) and (scheme, host) in trusted + + +def resolve_parallel_ai_credentials(api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + """ + Resolve the effective (api_base, api_key) pair for a Parallel AI LLM request. + + A server-managed key (from env) is only used when the request targets the + provider default or the operator's own PARALLEL_AI_API_BASE override; a + caller-supplied base must bring its own key, so the server credential is + never forwarded to a caller-chosen host. + """ + env_api_base: Final = get_secret_str("PARALLEL_AI_API_BASE") + resolved_api_base: Final = api_base or env_api_base or PARALLEL_AI_API_BASE + + if api_key: + return resolved_api_base, api_key + + server_api_key: Final = get_secret_str("PARALLEL_AI_API_KEY") or get_secret_str("PARALLEL_API_KEY") + if server_api_key and api_base and not _is_trusted_api_base(api_base, env_api_base): + raise ValueError( + f"Refusing to send the server-configured Parallel AI key to the caller-supplied " + f"api_base '{api_base}'. Pass an explicit api_key when overriding api_base." + ) + return resolved_api_base, server_api_key diff --git a/litellm/llms/parallel_ai/responses/__init__.py b/litellm/llms/parallel_ai/responses/__init__.py new file mode 100644 index 00000000000..fc0b2a4595d --- /dev/null +++ b/litellm/llms/parallel_ai/responses/__init__.py @@ -0,0 +1,7 @@ +""" +Parallel AI Responses API module. +""" + +from litellm.llms.parallel_ai.responses.transformation import ParallelAIResponsesConfig + +__all__ = ("ParallelAIResponsesConfig",) diff --git a/litellm/llms/parallel_ai/responses/cost_calculator.py b/litellm/llms/parallel_ai/responses/cost_calculator.py new file mode 100644 index 00000000000..8e10655b158 --- /dev/null +++ b/litellm/llms/parallel_ai/responses/cost_calculator.py @@ -0,0 +1,41 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +PARALLEL_AI_DEFAULT_REASONING_EFFORT: Final[str] = "medium" +PARALLEL_AI_EFFORT_TIER_MODELS: Final[Mapping[str, str]] = MappingProxyType( + { + "parallel-low": "low", + "parallel-medium": "medium", + "parallel-high": "high", + } +) +PARALLEL_AI_REASONING_EFFORTS: Final[frozenset[str]] = frozenset(PARALLEL_AI_EFFORT_TIER_MODELS.values()) +REASONING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def is_parallel_ai_response_model(model: str) -> bool: + model_without_provider: Final[str] = model.removeprefix("parallel_ai/") + return model_without_provider == "parallel" or model_without_provider in PARALLEL_AI_EFFORT_TIER_MODELS + + +def _reasoning_effort(optional_params: Mapping[str, object]) -> str: + try: + reasoning: Final = REASONING_ADAPTER.validate_python(optional_params.get("reasoning")) + except ValidationError: + return PARALLEL_AI_DEFAULT_REASONING_EFFORT + effort: Final[object] = reasoning.get("effort") + if isinstance(effort, str) and effort in PARALLEL_AI_REASONING_EFFORTS: + return effort + return PARALLEL_AI_DEFAULT_REASONING_EFFORT + + +def parallel_ai_response_pricing_model(model: str, optional_params: Mapping[str, object]) -> str: + """Return the cost-map model matching the effective Responses reasoning effort.""" + model_without_provider: Final[str] = model.removeprefix("parallel_ai/") + effort: Final[str] = PARALLEL_AI_EFFORT_TIER_MODELS.get(model_without_provider) or _reasoning_effort( + optional_params + ) + return f"parallel_ai/parallel-{effort}" diff --git a/litellm/llms/parallel_ai/responses/transformation.py b/litellm/llms/parallel_ai/responses/transformation.py new file mode 100644 index 00000000000..8fbc1bca322 --- /dev/null +++ b/litellm/llms/parallel_ai/responses/transformation.py @@ -0,0 +1,120 @@ +""" +Parallel AI Responses API, an OpenAI Responses-compatible web-research endpoint. + +Provider quirks: +- single `parallel` model; the performance tier is selected via `reasoning.effort` (low/medium/high), + or with the `parallel-low` / `parallel-medium` / `parallel-high` aliases, which pin the tier and + carry per-tier pricing in the cost map +- no `tools` param; web grounding is built in + +Ref: https://docs.parallel.ai/responses-api/responses-quickstart +""" + +from typing import Final + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials +from litellm.llms.parallel_ai.responses.cost_calculator import ( + PARALLEL_AI_EFFORT_TIER_MODELS as EFFORT_TIER_MODELS, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class ParallelAIResponsesConfig(OpenAIResponsesAPIConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseResponsesAPIConfig contract + """Ref: https://docs.parallel.ai/responses-api/responses-quickstart""" + return [ # mutable-ok: callers concatenate with other lists + "stream", + "reasoning", + "instructions", + "text", + "previous_response_id", + ] + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.PARALLEL_AI + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: BaseResponsesAPIConfig contract + """Parallel rejects unknown Responses params (e.g. `tools`), so unsupported keys are filtered out.""" + supported: Final = frozenset(self.get_supported_openai_params(model)) + return { # mutable-ok: request payload, merged downstream + key: value for key, value in response_api_optional_params.items() if key in supported + } + + def validate_environment( # mutable-ok: BaseResponsesAPIConfig contract + self, + headers: dict, # mutable-ok: BaseResponsesAPIConfig contract + model: str, + litellm_params: GenericLiteLLMParams | None, + ) -> dict: # mutable-ok: BaseResponsesAPIConfig contract + resolved_params: Final = litellm_params or GenericLiteLLMParams() + _, api_key = resolve_parallel_ai_credentials(api_base=resolved_params.api_base, api_key=resolved_params.api_key) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" # rebind-ok: stamping the caller's headers is the contract + return headers + + def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: # mutable-ok: base contract + resolved_api_base: Final = api_base or get_secret_str("PARALLEL_AI_API_BASE") or "https://api.parallel.ai" + trimmed: Final = resolved_api_base.rstrip("/") + if trimmed.endswith("/v1/responses"): + return trimmed + return f"{trimmed.removesuffix('/v1')}/v1/responses" + + def transform_responses_api_request( # mutable-ok: BaseResponsesAPIConfig contract + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict, # mutable-ok: BaseResponsesAPIConfig contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseResponsesAPIConfig contract + ) -> dict: # mutable-ok: BaseResponsesAPIConfig contract + """The tier aliases pin `reasoning.effort` so each alias bills at its cost map entry.""" + effort: Final = EFFORT_TIER_MODELS.get(model) + if effort is None: + return super().transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + return super().transform_responses_api_request( + model="parallel", + input=input, + response_api_optional_request_params={ # mutable-ok: request payload + **response_api_optional_request_params, + "reasoning": {"effort": effort}, # mutable-ok: request payload + }, + litellm_params=litellm_params, + headers=headers, + ) + + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """The API reports model `parallel` for every tier; cost tracking prices by response model, so the alias is restored.""" + response: Final = super().transform_response_api_response( + model=model, raw_response=raw_response, logging_obj=logging_obj + ) + if model in EFFORT_TIER_MODELS: + response.model = model + return response + + def supports_native_websocket(self) -> bool: + """Parallel AI does not support native WebSocket for the Responses API""" + return False diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a1961136d11..e2bd65a1a95 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -35282,6 +35282,42 @@ "output_cost_per_token": 1.25e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, + "parallel_ai/parallel": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-high": { + "input_cost_per_request": 0.25, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-low": { + "input_cost_per_request": 0.01, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-medium": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, "parallel_ai/search": { "input_cost_per_query": 0.004, "litellm_provider": "parallel_ai", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 80f56904367..d00fc21e7fc 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3797,6 +3797,7 @@ class LlmProviders(str, Enum): CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + PARALLEL_AI = "parallel_ai" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 92f7fde868a..7c772fc4db6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6334,6 +6334,11 @@ def validate_environment( keys_in_environment = True else: missing_keys.append("PERPLEXITYAI_API_KEY") + elif custom_llm_provider == "parallel_ai": + if "PARALLEL_AI_API_KEY" in os.environ or "PARALLEL_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("PARALLEL_AI_API_KEY") elif custom_llm_provider == "voyage": if "VOYAGE_API_KEY" in os.environ: keys_in_environment = True @@ -8487,6 +8492,8 @@ class ProviderConfigManager: return litellm.ManusResponsesAPIConfig() elif litellm.LlmProviders.PERPLEXITY == provider: return litellm.PerplexityResponsesConfig() + elif litellm.LlmProviders.PARALLEL_AI == provider: + return litellm.ParallelAIResponsesConfig() elif litellm.LlmProviders.DATABRICKS == provider: # Databricks Responses API is only compatible with OpenAI GPT models if model and "gpt" in model.lower(): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a1961136d11..e2bd65a1a95 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -35282,6 +35282,42 @@ "output_cost_per_token": 1.25e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, + "parallel_ai/parallel": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-high": { + "input_cost_per_request": 0.25, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-low": { + "input_cost_per_request": 0.01, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-medium": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, "parallel_ai/search": { "input_cost_per_query": 0.004, "litellm_provider": "parallel_ai", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7c1ca34c23c..42686610afb 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1934,9 +1934,9 @@ "display_name": "Parallel AI (`parallel_ai`)", "url": "https://docs.litellm.ai/docs/search/parallel_ai", "endpoints": { - "chat_completions": false, - "messages": false, - "responses": false, + "chat_completions": true, + "messages": true, + "responses": true, "embeddings": false, "image_generations": false, "audio_transcriptions": false, diff --git a/tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py b/tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py new file mode 100644 index 00000000000..cdbb017e512 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py @@ -0,0 +1,401 @@ +""" +Tests for Parallel AI Responses API transformation. + +Source: litellm/llms/parallel_ai/responses/transformation.py +""" + +import json +import pytest + +from litellm.llms.parallel_ai.responses.cost_calculator import parallel_ai_response_pricing_model +from litellm.llms.parallel_ai.responses.transformation import ParallelAIResponsesConfig +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +def _parallel_response_body(response_id: str = "resp_test") -> dict[str, object]: + return { + "id": response_id, + "object": "response", + "created_at": 1700000000, + "model": "parallel", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "grounded answer", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +class TestParallelAIResponsesConfig: + def test_provider_config_manager_returns_parallel_config(self): + config = ProviderConfigManager.get_provider_responses_api_config( + provider=LlmProviders.PARALLEL_AI, model="parallel" + ) + assert isinstance(config, ParallelAIResponsesConfig) + + @pytest.mark.parametrize( + "api_base,expected", + [ + (None, "https://api.parallel.ai/v1/responses"), + ("https://api.parallel.ai", "https://api.parallel.ai/v1/responses"), + ("https://api.parallel.ai/", "https://api.parallel.ai/v1/responses"), + ("https://api.parallel.ai/v1", "https://api.parallel.ai/v1/responses"), + ("https://proxy.example.com/v1/responses", "https://proxy.example.com/v1/responses"), + ], + ) + def test_get_complete_url(self, api_base, expected, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + config = ParallelAIResponsesConfig() + assert config.get_complete_url(api_base=api_base, litellm_params={}) == expected + + def test_validate_environment_sets_bearer_from_env(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + config = ParallelAIResponsesConfig() + + headers = config.validate_environment(headers={}, model="parallel", litellm_params=None) + assert headers["Authorization"] == "Bearer pk-test" + + def test_validate_environment_prefers_explicit_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + config = ParallelAIResponsesConfig() + + headers = config.validate_environment( + headers={}, + model="parallel", + litellm_params=GenericLiteLLMParams(api_key="pk-explicit"), + ) + assert headers["Authorization"] == "Bearer pk-explicit" + + def test_reasoning_effort_is_supported(self): + config = ParallelAIResponsesConfig() + supported = config.get_supported_openai_params(model="parallel") + assert "reasoning" in supported + assert "instructions" in supported + assert "previous_response_id" in supported + assert "tools" not in supported + + def test_unsupported_params_dropped_with_drop_params(self): + from litellm.responses.utils import ResponsesAPIRequestUtils + + config = ParallelAIResponsesConfig() + params = ResponsesAPIOptionalRequestParams( + reasoning={"effort": "high"}, + tools=[{"type": "web_search"}], + temperature=0.5, + ) + + mapped = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="parallel", + responses_api_provider_config=config, + response_api_optional_params=params, + drop_params=True, + ) + assert mapped["reasoning"] == {"effort": "high"} + assert "tools" not in mapped + assert "temperature" not in mapped + + def test_unsupported_params_raise_without_drop_params(self): + import litellm as litellm_module + from litellm.responses.utils import ResponsesAPIRequestUtils + + config = ParallelAIResponsesConfig() + params = ResponsesAPIOptionalRequestParams(tools=[{"type": "web_search"}]) + + with pytest.raises(litellm_module.UnsupportedParamsError): + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="parallel", + responses_api_provider_config=config, + response_api_optional_params=params, + drop_params=False, + ) + + def test_transform_request_sends_parallel_model(self): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model="parallel", + input="What is the latest AI news?", + response_api_optional_request_params={"reasoning": {"effort": "low"}}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["model"] == "parallel" + assert request["input"] == "What is the latest AI news?" + assert request["reasoning"] == {"effort": "low"} + + def test_no_native_websocket(self): + assert ParallelAIResponsesConfig().supports_native_websocket() is False + + def test_untrusted_api_base_refuses_server_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + config = ParallelAIResponsesConfig() + + with pytest.raises(ValueError, match="Refusing to send"): + config.validate_environment( + headers={}, + model="parallel", + litellm_params=GenericLiteLLMParams(api_base="https://attacker.example.com"), + ) + + def test_untrusted_api_base_with_explicit_key_is_allowed(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + config = ParallelAIResponsesConfig() + + headers = config.validate_environment( + headers={}, + model="parallel", + litellm_params=GenericLiteLLMParams(api_base="https://proxy.example.com", api_key="pk-caller"), + ) + assert headers["Authorization"] == "Bearer pk-caller" + + +class TestParallelAICompletionBridge: + @pytest.mark.respx() + def test_completion_routes_through_responses_api(self, respx_mock, monkeypatch): + """litellm.completion() on a mode=responses model must bridge to /v1/responses.""" + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + import litellm as litellm_module + + litellm_module.model_cost = litellm_module.get_model_cost_map(url="") + + respx_mock.post("https://api.parallel.ai/v1/responses").respond(json=_parallel_response_body()) + + response = litellm_module.completion( + model="parallel_ai/parallel", + messages=[{"role": "user", "content": "question"}], + ) + + assert response.choices[0].message.content == "grounded answer" + + +class TestParallelAIEffortTierAliases: + @pytest.mark.parametrize( + "alias,effort", + [("parallel-low", "low"), ("parallel-medium", "medium"), ("parallel-high", "high")], + ) + def test_alias_pins_model_and_effort(self, alias, effort): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model=alias, + input="question", + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["model"] == "parallel" + assert request["reasoning"] == {"effort": effort} + + def test_alias_effort_wins_over_explicit_reasoning(self): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model="parallel-high", + input="question", + response_api_optional_request_params={"reasoning": {"effort": "low"}}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["reasoning"] == {"effort": "high"} + + def test_plain_model_passes_reasoning_through(self): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model="parallel", + input="question", + response_api_optional_request_params={"reasoning": {"effort": "low"}}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["model"] == "parallel" + assert request["reasoning"] == {"effort": "low"} + + def test_alias_restored_on_response_model_for_cost_tracking(self): + from unittest.mock import MagicMock + + config = ParallelAIResponsesConfig() + raw = MagicMock() + raw.json.return_value = { + "id": "resp_1", + "object": "response", + "created_at": 1700000000, + "model": "parallel", + "status": "completed", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + response = config.transform_response_api_response( + model="parallel-low", raw_response=raw, logging_obj=MagicMock() + ) + assert response.model == "parallel-low" + + raw.json.return_value["model"] = "parallel" + response_plain = config.transform_response_api_response( + model="parallel", raw_response=raw, logging_obj=MagicMock() + ) + assert response_plain.model == "parallel" + + +class TestParallelAIReasoningEffortPricing: + @pytest.mark.parametrize( + "model,optional_params,expected_model", + [ + ("parallel", {"reasoning": {"effort": "low"}}, "parallel_ai/parallel-low"), + ("parallel", {}, "parallel_ai/parallel-medium"), + ("parallel", {"reasoning": {"effort": "high"}}, "parallel_ai/parallel-high"), + ("parallel_ai/parallel", {"reasoning": {"effort": "low"}}, "parallel_ai/parallel-low"), + ("parallel-high", {"reasoning": {"effort": "low"}}, "parallel_ai/parallel-high"), + ], + ) + def test_pricing_model_uses_effective_reasoning_effort(self, model, optional_params, expected_model): + assert parallel_ai_response_pricing_model(model=model, optional_params=optional_params) == expected_model + + @pytest.mark.parametrize( + "model,optional_params,expected_cost", + [ + ("parallel_ai/parallel", {"reasoning": {"effort": "low"}}, 0.01), + ("parallel_ai/parallel", {}, 0.05), + ("parallel_ai/parallel", {"reasoning": {"effort": "high"}}, 0.25), + ("parallel_ai/parallel-high", {"reasoning": {"effort": "low"}}, 0.25), + ], + ) + def test_completion_cost_uses_effective_reasoning_effort(self, model, optional_params, expected_cost, monkeypatch): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + response = ResponsesAPIResponse( + id="resp_cost", + created_at=1700000000, + model="parallel", + output=[], + usage={"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + ) + + cost = litellm_module.completion_cost( + completion_response=response, + model=model, + optional_params=optional_params, + call_type="responses", + ) + + assert cost == pytest.approx(expected_cost) + + def test_explicit_base_model_remains_authoritative(self, monkeypatch): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + response = ResponsesAPIResponse( + id="resp_custom_base", + created_at=1700000000, + model="parallel", + output=[], + usage={"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + ) + + cost = litellm_module.completion_cost( + completion_response=response, + model="parallel_ai/parallel", + optional_params={"reasoning": {"effort": "high"}}, + base_model="parallel_ai/parallel-low", + call_type="responses", + ) + + assert cost == pytest.approx(0.01) + + @pytest.mark.respx() + @pytest.mark.parametrize( + "reasoning,expected_cost", + [ + ({"effort": "low"}, 0.01), + (None, 0.05), + ({"effort": "high"}, 0.25), + ], + ) + def test_responses_records_effort_aware_cost(self, reasoning, expected_cost, respx_mock, monkeypatch): + import litellm as litellm_module + + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + route = respx_mock.post("https://api.parallel.ai/v1/responses").respond(json=_parallel_response_body()) + + response = litellm_module.responses( + model="parallel_ai/parallel", + input="question", + reasoning=reasoning, + ) + + assert response.model == "parallel" + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) + request_body = json.loads(route.calls.last.request.content) + if reasoning is None: + assert "reasoning" not in request_body + else: + assert request_body["reasoning"] == reasoning + + @pytest.mark.respx() + def test_streaming_response_records_high_effort_cost(self, respx_mock, monkeypatch): + import litellm as litellm_module + + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + monkeypatch.setattr(litellm_module, "include_cost_in_streaming_usage", True) + completed_event = { + "type": "response.completed", + "response": _parallel_response_body(response_id="resp_stream"), + } + stream_body = f"data: {json.dumps(completed_event)}\n\ndata: [DONE]\n\n".encode() + respx_mock.post("https://api.parallel.ai/v1/responses").respond( + content=stream_body, + headers={"content-type": "text/event-stream"}, + ) + + stream = litellm_module.responses( + model="parallel_ai/parallel", + input="question", + reasoning={"effort": "high"}, + stream=True, + ) + events = list(stream) + + assert len(events) == 1 + assert events[0].response.model == "parallel" + assert events[0].response.usage.cost == pytest.approx(0.25) + + @pytest.mark.respx() + def test_completion_bridge_records_high_effort_cost(self, respx_mock, monkeypatch): + import litellm as litellm_module + + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + route = respx_mock.post("https://api.parallel.ai/v1/responses").respond(json=_parallel_response_body()) + + response = litellm_module.completion( + model="parallel_ai/parallel", + messages=[{"role": "user", "content": "question"}], + reasoning={"effort": "high"}, + ) + + assert response.choices[0].message.content == "grounded answer" + assert response._hidden_params["response_cost"] == pytest.approx(0.25) + request_body = json.loads(route.calls.last.request.content) + assert request_body["reasoning"] == {"effort": "high"} diff --git a/tests/test_litellm/llms/parallel_ai/test_common_utils.py b/tests/test_litellm/llms/parallel_ai/test_common_utils.py new file mode 100644 index 00000000000..e70ea8b557a --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/test_common_utils.py @@ -0,0 +1,123 @@ +""" +Tests for Parallel AI credential and provider resolution. + +Source: litellm/llms/parallel_ai/common_utils.py +""" + +import pytest + +import litellm +from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials + + +class TestResolveParallelAICredentials: + @pytest.fixture(autouse=True) + def _clean_env(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + + def test_defaults(self, monkeypatch): + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + + api_base, api_key = resolve_parallel_ai_credentials(api_base=None, api_key=None) + assert api_base == "https://api.parallel.ai" + assert api_key == "pk-test" + + def test_prefers_parallel_ai_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-primary") + monkeypatch.setenv("PARALLEL_API_KEY", "pk-fallback") + + _, api_key = resolve_parallel_ai_credentials(api_base=None, api_key=None) + assert api_key == "pk-primary" + + def test_plaintext_provider_host_is_not_trusted(self, monkeypatch): + """A host-only trust check would send the server key over plaintext HTTP.""" + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + with pytest.raises(ValueError, match="Refusing to send"): + resolve_parallel_ai_credentials(api_base="http://api.parallel.ai", api_key=None) + + def test_https_provider_host_is_trusted(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://api.parallel.ai", api_key=None) + assert api_base == "https://api.parallel.ai" + assert api_key == "pk-env" + + def test_operator_plaintext_override_is_trusted(self, monkeypatch): + """The operator's own PARALLEL_AI_API_BASE is trusted at the scheme it names.""" + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + monkeypatch.setenv("PARALLEL_AI_API_BASE", "http://parallel-proxy.internal") + + _, api_key = resolve_parallel_ai_credentials(api_base="http://parallel-proxy.internal", api_key=None) + assert api_key == "pk-env" + + def test_explicit_args_win(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://proxy.example.com", api_key="pk-explicit") + assert api_base == "https://proxy.example.com" + assert api_key == "pk-explicit" + + def test_untrusted_api_base_refuses_server_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + + with pytest.raises(ValueError, match="Refusing to send"): + resolve_parallel_ai_credentials(api_base="https://attacker.example.com", api_key=None) + + def test_operator_env_base_is_trusted(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://proxy.internal.example.com") + + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://proxy.internal.example.com", api_key=None) + assert api_base == "https://proxy.internal.example.com" + assert api_key == "pk-server-secret" + + def test_untrusted_base_without_server_key_returns_none(self): + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://proxy.example.com", api_key=None) + assert api_base == "https://proxy.example.com" + assert api_key is None + + +class TestParallelAIProviderWiring: + def test_get_llm_provider_routes_parallel_ai(self, monkeypatch): + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + model, provider, api_key, api_base = litellm.get_llm_provider("parallel_ai/parallel") + assert model == "parallel" + assert provider == "parallel_ai" + assert api_key == "pk-test" + assert api_base == "https://api.parallel.ai" + + def test_get_llm_provider_detects_parallel_api_base(self, monkeypatch): + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + + model, provider, api_key, api_base = litellm.get_llm_provider( + model="parallel", api_base="https://api.parallel.ai" + ) + assert provider == "parallel_ai" + assert api_key == "pk-test" + + def test_validate_environment_reports_missing_key(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + + result = litellm.validate_environment(model="parallel_ai/parallel") + assert result["keys_in_environment"] is False + assert "PARALLEL_AI_API_KEY" in result["missing_keys"] + + def test_validate_environment_accepts_either_key_name(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + + result = litellm.validate_environment(model="parallel_ai/parallel") + assert result["keys_in_environment"] is True + + def test_api_base_detection_keeps_explicit_caller_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + model, provider, api_key, api_base = litellm.get_llm_provider( + model="parallel", api_base="https://api.parallel.ai", api_key="pk-explicit" + ) + assert provider == "parallel_ai" + assert api_key == "pk-explicit" diff --git a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py new file mode 100644 index 00000000000..30f93e85676 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py @@ -0,0 +1,138 @@ +"""Gateway coverage for Parallel AI's Responses integration.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Final +from unittest.mock import AsyncMock + +import httpx +import pytest +from fastapi.testclient import TestClient + +import litellm +from litellm import Router +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +PARALLEL_RESPONSES_URL: Final = "https://api.parallel.ai/v1/responses" + + +@pytest.fixture +def client() -> TestClient: + return TestClient(proxy_server.app, raise_server_exceptions=False) + + +@pytest.fixture +def auth_as() -> Iterator[None]: + async def _authorized_request() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-sk-test", + user_id="parallel-test-user", + ) + + previous: Final = proxy_server.app.dependency_overrides.get(user_api_key_auth) + proxy_server.app.dependency_overrides[user_api_key_auth] = _authorized_request + try: + yield + finally: + if previous is None: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + else: + proxy_server.app.dependency_overrides[user_api_key_auth] = previous + + +def _parallel_response_body() -> dict[str, object]: + return { + "id": "resp_parallel_gateway", + "object": "response", + "created_at": 1700000000, + "model": "parallel", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_parallel_gateway", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Parallel grounded answer", + "annotations": [], + } + ], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +def _parallel_router() -> Router: + return Router( + model_list=[ + { + "model_name": "parallel-gateway", + "litellm_params": { + "model": "parallel_ai/parallel", + "api_key": "parallel-responses-key", + "use_in_pass_through": True, + }, + } + ], + num_retries=0, + ) + + +def _mock_async_post( + monkeypatch, + *, + url: str, + response_body: dict[str, object], +) -> AsyncMock: + response = httpx.Response( + status_code=200, + json=response_body, + request=httpx.Request("POST", url), + ) + mock_post = AsyncMock(return_value=response) + monkeypatch.setattr(AsyncHTTPHandler, "post", mock_post) + return mock_post + + +def test_parallel_responses_gateway_route(client, auth_as, monkeypatch): + """The primary LLM route reaches Parallel's native Responses endpoint.""" + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", _parallel_router()) + mock_post = _mock_async_post( + monkeypatch, + url=PARALLEL_RESPONSES_URL, + response_body=_parallel_response_body(), + ) + + response = client.post( + "/v1/responses", + json={"model": "parallel-gateway", "input": "Research Parallel"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["output"][0]["content"][0]["text"] == "Parallel grounded answer" + + request_kwargs = mock_post.await_args.kwargs + assert request_kwargs["url"] == PARALLEL_RESPONSES_URL + assert request_kwargs["headers"]["Authorization"] == "Bearer parallel-responses-key" + assert request_kwargs["json"] == { + "model": "parallel", + "input": "Research Parallel", + } From 52e7ea43325a9b1ebcea1340ec98046242c09c19 Mon Sep 17 00:00:00 2001 From: James Liounis Date: Fri, 21 Aug 2026 16:08:28 -0400 Subject: [PATCH 4/5] feat(parallel_ai): add a native Extract pass-through with cost tracking Parallel's /v1/extract endpoint has no equivalent in any unified LiteLLM surface: it takes a URL list and returns per-URL excerpts and full content, with partial success reported as a parallel errors array. Forcing it into the search or responses shape would lose that response. Adds it as a pass-through at /parallel_ai/v1/extract so the request and response cross unchanged, with the gateway supplying the provider credential. The success handler prices the call from the URLs the provider reports as billed, falling back to the requested URL count when usage is absent, so extract traffic is attributed rather than logged at $0. Adding a route adds one FastAPI Depends() default, so the B008 ceiling moves by one. --- litellm/llms/parallel_ai/extract/__init__.py | 6 + .../parallel_ai/extract/cost_calculator.py | 73 +++++++ litellm/proxy/_types.py | 1 + .../llm_passthrough_endpoints.py | 47 +++++ ...parallel_ai_passthrough_logging_handler.py | 51 +++++ .../pass_through_endpoints/success_handler.py | 13 ++ ruff-strict-budget.json | 2 +- .../extract/test_cost_calculator.py | 90 +++++++++ .../test_parallel_ai_extract_gateway.py | 185 ++++++++++++++++++ ...parallel_ai_passthrough_logging_handler.py | 133 +++++++++++++ 10 files changed, 600 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/parallel_ai/extract/__init__.py create mode 100644 litellm/llms/parallel_ai/extract/cost_calculator.py create mode 100644 litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py create mode 100644 tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py create mode 100644 tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py create mode 100644 tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py diff --git a/litellm/llms/parallel_ai/extract/__init__.py b/litellm/llms/parallel_ai/extract/__init__.py new file mode 100644 index 00000000000..b3d6df3959b --- /dev/null +++ b/litellm/llms/parallel_ai/extract/__init__.py @@ -0,0 +1,6 @@ +from litellm.llms.parallel_ai.extract.cost_calculator import ( + PARALLEL_AI_EXTRACT_MODEL, + parallel_ai_extract_cost, +) + +__all__ = ["PARALLEL_AI_EXTRACT_MODEL", "parallel_ai_extract_cost"] diff --git a/litellm/llms/parallel_ai/extract/cost_calculator.py b/litellm/llms/parallel_ai/extract/cost_calculator.py new file mode 100644 index 00000000000..7ec090e2029 --- /dev/null +++ b/litellm/llms/parallel_ai/extract/cost_calculator.py @@ -0,0 +1,73 @@ +from typing import Annotated, Final + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, ValidationError + +PARALLEL_AI_EXTRACT_COST_PER_URL: Final = 0.001 +PARALLEL_AI_EXTRACT_MODEL: Final = "parallel_ai/extract" +PARALLEL_AI_EXTRACT_USAGE_SKU: Final = "sku_extract_excerpts" + + +class _ParallelAIExtractUsageItem(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: StrictStr + count: Annotated[StrictInt, Field(ge=0)] + + +class _ParallelAIExtractUsageName(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: StrictStr + + +class _ParallelAIExtractBillingResponse(BaseModel): + model_config = ConfigDict(extra="ignore") + + usage: tuple[object, ...] | None = None + + +class _ParallelAIExtractBillingRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + urls: tuple[StrictStr, ...] = () + + +def _usage_url_count(response_body: object) -> int | None: + try: + parsed: Final = _ParallelAIExtractBillingResponse.model_validate(response_body) + except ValidationError: + return None + if parsed.usage is None: + return None + + target_items: Final = tuple(item for item in parsed.usage if _usage_name(item) == PARALLEL_AI_EXTRACT_USAGE_SKU) + if not target_items: + return 0 + + try: + usage_items: Final = tuple(_ParallelAIExtractUsageItem.model_validate(item) for item in target_items) + except ValidationError: + return None + return sum(item.count for item in usage_items) + + +def _usage_name(usage_item: object) -> str | None: + try: + parsed: Final = _ParallelAIExtractUsageName.model_validate(usage_item) + except ValidationError: + return None + return parsed.name + + +def _request_url_count(request_body: object) -> int: + try: + parsed: Final = _ParallelAIExtractBillingRequest.model_validate(request_body) + except ValidationError: + return 0 + return len(parsed.urls) + + +def parallel_ai_extract_cost(request_body: object, response_body: object) -> float: + usage_url_count: Final = _usage_url_count(response_body) + billed_url_count: Final = usage_url_count if usage_url_count is not None else _request_url_count(request_body) + return billed_url_count * PARALLEL_AI_EXTRACT_COST_PER_URL diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e51a3138d2d..546bddab883 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -466,6 +466,7 @@ class LiteLLMRoutes(enum.Enum): "/openai_passthrough", "/assemblyai", "/eu.assemblyai", + "/parallel_ai", "/vllm", "/mistral", "/milvus", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 7ce41c1d5b6..3ce62b2f7f7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -26,6 +26,7 @@ from litellm.constants import ( BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks @@ -310,6 +311,52 @@ async def cohere_proxy_route( return received_value +def _parallel_ai_extract_url(api_base: str) -> str: + trimmed: Final = api_base.rstrip("/") + if trimmed.endswith("/v1/extract"): + return trimmed + return f"{trimmed.removesuffix('/v1')}/v1/extract" + + +@router.post( + "/parallel_ai/v1/extract", + tags=["Parallel AI Pass-through", "pass-through"], +) +async def parallel_ai_extract_proxy_route( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + deployment_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="parallel_ai", + region_name=None, + ) + api_base, api_key = resolve_parallel_ai_credentials( + api_base=None, + api_key=deployment_api_key, + ) + if api_key is None: + raise HTTPException( + status_code=500, + detail="PARALLEL_AI_API_KEY or PARALLEL_API_KEY is required for the Parallel AI Extract pass-through.", + ) + + endpoint_func: Final = create_pass_through_route( + endpoint="/parallel_ai/v1/extract", + target=_parallel_ai_extract_url(api_base), + custom_headers={ + "Content-Type": "application/json", + "x-api-key": api_key, + }, + custom_llm_provider="parallel_ai", + ) + return await endpoint_func( + request, + fastapi_response, + user_api_key_dict, + ) + + @router.api_route( "/vllm/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..1d98e647702 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py @@ -0,0 +1,51 @@ +import json +from collections.abc import Mapping +from typing import Final +from urllib.parse import urlparse + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.parallel_ai.extract.cost_calculator import ( + PARALLEL_AI_EXTRACT_MODEL, + parallel_ai_extract_cost, +) +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.utils import StandardPassThroughResponseObject + + +class ParallelAIPassthroughLoggingHandler: + @staticmethod + def is_extract_route(url_route: str, custom_llm_provider: str | None) -> bool: + path: Final = urlparse(url_route).path.rstrip("/") + return custom_llm_provider == "parallel_ai" and path.endswith("/v1/extract") + + @staticmethod + def parallel_ai_extract_handler( + response_body: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + request_body: Mapping[str, object], + **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler + ) -> PassThroughEndpointLoggingTypedDict: + """ + Prices a Parallel AI Extract call from the URLs the provider reports as + billed (falling back to the requested URL count) and records model, + provider, and cost on the logging payload. + """ + response_cost: Final = parallel_ai_extract_cost( + request_body=request_body, + response_body=response_body, + ) + logging_obj.model_call_details.update( + model=PARALLEL_AI_EXTRACT_MODEL, + custom_llm_provider="parallel_ai", + response_cost=response_cost, + ) + + return { + "result": StandardPassThroughResponseObject(response=json.dumps(response_body)), + "kwargs": { # mutable-ok: the logging pipeline requires a plain kwargs dict + **kwargs, + "model": PARALLEL_AI_EXTRACT_MODEL, + "custom_llm_provider": "parallel_ai", + "response_cost": response_cost, + }, + } diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index c38566375f4..1553364bd4b 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -27,6 +27,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import ( from .llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) +from .llm_provider_handlers.parallel_ai_passthrough_logging_handler import ( + ParallelAIPassthroughLoggingHandler, +) from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) @@ -221,6 +224,16 @@ class PassThroughEndpointLogging: standard_logging_response_object = openai_passthrough_logging_handler_result["result"] kwargs = openai_passthrough_logging_handler_result["kwargs"] + elif ParallelAIPassthroughLoggingHandler.is_extract_route(url_route, custom_llm_provider): + parallel_ai_result = ParallelAIPassthroughLoggingHandler.parallel_ai_extract_handler( + response_body=response_body if isinstance(response_body, dict) else {}, + logging_obj=logging_obj, + request_body=request_body, + **kwargs, + ) + standard_logging_response_object = parallel_ai_result["result"] + kwargs = parallel_ai_result["kwargs"] # rebind-ok: every dispatch branch reassigns kwargs + elif self.is_cursor_route(url_route, custom_llm_provider): cursor_passthrough_logging_handler_result = CursorPassthroughLoggingHandler.cursor_passthrough_handler( httpx_response=httpx_response, diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index a990f7c3830..7b3b67146c8 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -36,7 +36,7 @@ "limit": 177 }, "B008": { - "limit": 503 + "limit": 505 }, "B009": { "limit": 59 diff --git a/tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py b/tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py new file mode 100644 index 00000000000..83d8532da14 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py @@ -0,0 +1,90 @@ +import pytest + +from litellm.llms.parallel_ai.extract.cost_calculator import parallel_ai_extract_cost + + +def test_extract_cost_prefers_provider_usage_over_requested_urls() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2", "https://example.com/3"]}, + response_body={"usage": [{"name": "sku_extract_excerpts", "count": 2}]}, + ) + + assert cost == pytest.approx(0.002) + + +def test_extract_cost_sums_repeated_usage_skus() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1"]}, + response_body={ + "usage": [ + {"name": "sku_extract_excerpts", "count": 1}, + {"name": "unrelated_sku", "count": 9}, + {"name": "sku_extract_excerpts", "count": 2}, + ] + }, + ) + + assert cost == pytest.approx(0.003) + + +def test_extract_cost_ignores_malformed_unrelated_usage_skus() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={ + "usage": [ + {"name": "sku_extract_excerpts", "count": 1}, + {"name": "unrelated_sku", "count": "not-an-integer"}, + ] + }, + ) + + assert cost == pytest.approx(0.001) + + +def test_extract_cost_treats_usage_without_extract_sku_as_unbilled() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"usage": [{"name": "unrelated_sku", "count": 2}]}, + ) + + assert cost == 0.0 + + +def test_extract_cost_treats_empty_usage_as_unbilled() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"usage": []}, + ) + + assert cost == 0.0 + + +def test_extract_cost_falls_back_to_requested_url_count_without_usage() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"results": []}, + ) + + assert cost == pytest.approx(0.002) + + +@pytest.mark.parametrize("invalid_count", [True, -1, "2"]) +def test_extract_cost_falls_back_when_provider_usage_is_invalid(invalid_count: object) -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"usage": [{"name": "sku_extract_excerpts", "count": invalid_count}]}, + ) + + assert cost == pytest.approx(0.002) + + +@pytest.mark.parametrize( + "request_body", + [ + {}, + {"urls": "https://example.com"}, + {"urls": ["https://example.com", 42]}, + ], +) +def test_extract_cost_does_not_guess_from_invalid_request_urls(request_body: object) -> None: + assert parallel_ai_extract_cost(request_body=request_body, response_body={}) == 0.0 diff --git a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py new file mode 100644 index 00000000000..4df5f3d5333 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py @@ -0,0 +1,185 @@ +"""Gateway coverage for Parallel AI's Extract pass-through.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from typing import Final + +import pytest +from fastapi.testclient import TestClient + +import litellm +from litellm import Router +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +PARALLEL_EXTRACT_URL: Final = "https://api.parallel.ai/v1/extract" + + +@pytest.fixture +def client() -> TestClient: + return TestClient(proxy_server.app, raise_server_exceptions=False) + + +@pytest.fixture +def auth_as() -> Iterator[None]: + async def _authorized_request() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-sk-test", + user_id="parallel-test-user", + ) + + previous: Final = proxy_server.app.dependency_overrides.get(user_api_key_auth) + proxy_server.app.dependency_overrides[user_api_key_auth] = _authorized_request + try: + yield + finally: + if previous is None: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + else: + proxy_server.app.dependency_overrides[user_api_key_auth] = previous + + +def _parallel_extract_body() -> dict[str, object]: + return { + "extract_id": "extract_parallel_gateway", + "results": [ + { + "url": "https://example.com/parallel", + "title": "Parallel result", + "publish_date": "2026-08-14", + "excerpts": ["Focused excerpt"], + "full_content": "# Full content", + } + ], + "errors": [ + { + "url": "https://example.com/unavailable", + "error_type": "fetch_error", + "http_status_code": 503, + "content": "Upstream unavailable", + } + ], + "warnings": None, + "usage": [{"name": "sku_extract_excerpts", "count": 2}], + "session_id": "session_parallel_gateway", + } + + +def _parallel_router() -> Router: + return Router( + model_list=[ + { + "model_name": "parallel-gateway", + "litellm_params": { + "model": "parallel_ai/parallel", + "api_key": "parallel-responses-key", + "use_in_pass_through": True, + }, + } + ], + num_retries=0, + ) + + +def test_parallel_extract_gateway_route(client, auth_as, monkeypatch, respx_mock): + """The native Extract route preserves Parallel's V1 request and partial-success response.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", _parallel_router()) + upstream_route = respx_mock.post(PARALLEL_EXTRACT_URL).respond(json=_parallel_extract_body()) + request_body = { + "urls": [ + "https://example.com/parallel", + "https://example.com/unavailable", + ], + "objective": "Find the integration details", + "search_queries": ["Parallel integration"], + "max_chars_total": 50000, + "session_id": "session_parallel_gateway", + "client_model": "gpt-5.4", + "advanced_settings": { + "fetch_policy": { + "max_age_seconds": 3600, + "timeout_seconds": 30, + "disable_cache_fallback": False, + }, + "excerpt_settings": {"max_chars_per_result": 5000}, + "full_content": {"max_chars_per_result": 50000}, + }, + } + + response = client.post("/parallel_ai/v1/extract", json=request_body) + + assert response.status_code == 200, response.text + assert response.json() == _parallel_extract_body() + assert upstream_route.called + + upstream_request = upstream_route.calls.last.request + assert upstream_request.headers["x-api-key"] == "parallel-responses-key" + assert "authorization" not in upstream_request.headers + assert json.loads(upstream_request.content) == request_body + + +def test_parallel_extract_route_is_classified_as_an_llm_api_route() -> None: + assert RouteChecks.is_llm_api_route(route="/parallel_ai/v1/extract") is True + + +def test_parallel_extract_gateway_uses_environment_configuration(client, auth_as, monkeypatch, respx_mock): + custom_url = "https://parallel-proxy.example.com/v1/extract" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "parallel-env-key") + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://parallel-proxy.example.com/v1") + upstream_route = respx_mock.post(custom_url).respond(json=_parallel_extract_body()) + + response = client.post( + "/parallel_ai/v1/extract", + json={"urls": ["https://example.com/parallel"]}, + ) + + assert response.status_code == 200, response.text + assert upstream_route.called + assert upstream_route.calls.last.request.headers["x-api-key"] == "parallel-env-key" + + +def test_parallel_extract_gateway_requires_a_provider_key(client, auth_as, monkeypatch) -> None: + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + + response = client.post( + "/parallel_ai/v1/extract", + json={"urls": ["https://example.com/parallel"]}, + ) + + assert response.status_code == 500 + assert response.json()["detail"] == ( + "PARALLEL_AI_API_KEY or PARALLEL_API_KEY is required for the Parallel AI Extract pass-through." + ) + + +def test_parallel_extract_gateway_preserves_validation_errors(client, auth_as, monkeypatch, respx_mock): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", _parallel_router()) + error_body = { + "error": { + "type": "validation_error", + "message": "urls must contain at most 20 items", + } + } + respx_mock.post(PARALLEL_EXTRACT_URL).respond(status_code=422, json=error_body) + + response = client.post( + "/parallel_ai/v1/extract", + json={"urls": [f"https://example.com/{index}" for index in range(21)]}, + ) + + assert response.status_code == 422 + assert response.json() == error_body diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..431ab51b181 --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py @@ -0,0 +1,133 @@ +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.llms.parallel_ai.extract.cost_calculator import PARALLEL_AI_EXTRACT_MODEL +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.parallel_ai_passthrough_logging_handler import ( + ParallelAIPassthroughLoggingHandler, +) +from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging + + +def test_extract_route_detection_requires_parallel_provider() -> None: + assert ParallelAIPassthroughLoggingHandler.is_extract_route( + "https://api.parallel.ai/v1/extract", + "parallel_ai", + ) + assert not ParallelAIPassthroughLoggingHandler.is_extract_route( + "https://api.parallel.ai/v1/extract", + None, + ) + assert not ParallelAIPassthroughLoggingHandler.is_extract_route( + "https://api.parallel.ai/v1/search", + "parallel_ai", + ) + + +def test_extract_handler_sets_usage_aware_cost_and_model() -> None: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "parallel-extract-call" + logging_obj.model_call_details = {} + response_body = { + "extract_id": "extract_test", + "results": [], + "errors": [], + "usage": [{"name": "sku_extract_excerpts", "count": 2}], + "session_id": "session_test", + } + + result = ParallelAIPassthroughLoggingHandler.parallel_ai_extract_handler( + response_body=response_body, + logging_obj=logging_obj, + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + ) + + assert result["kwargs"]["model"] == PARALLEL_AI_EXTRACT_MODEL + assert result["kwargs"]["custom_llm_provider"] == "parallel_ai" + assert result["kwargs"]["response_cost"] == 0.002 + assert logging_obj.model_call_details["model"] == PARALLEL_AI_EXTRACT_MODEL + assert logging_obj.model_call_details["custom_llm_provider"] == "parallel_ai" + assert logging_obj.model_call_details["response_cost"] == 0.002 + + +def test_success_handler_dispatches_parallel_extract_billing() -> None: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "parallel-extract-dispatch" + logging_obj.model_call_details = {} + response_body = { + "extract_id": "extract_dispatch", + "results": [], + "errors": [], + "usage": [{"name": "sku_extract_excerpts", "count": 1}], + "session_id": "session_dispatch", + } + response = httpx.Response( + 200, + json=response_body, + request=httpx.Request("POST", "https://api.parallel.ai/v1/extract"), + ) + + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=response, + response_body=response_body, + request_body={"urls": ["https://example.com"]}, + logging_obj=logging_obj, + url_route="https://api.parallel.ai/v1/extract", + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="parallel_ai", + ) + + assert normalized["standard_logging_response_object"] is not None + assert normalized["kwargs"]["model"] == PARALLEL_AI_EXTRACT_MODEL + assert normalized["kwargs"]["response_cost"] == 0.001 + + +@pytest.mark.asyncio +async def test_success_handler_sends_extract_cost_to_async_loggers() -> None: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "parallel-extract-logging" + logging_obj.model_call_details = {} + logging_obj.dispatch_success_handlers = AsyncMock() + response_body = { + "extract_id": "extract_logging", + "results": [], + "errors": [], + "usage": [{"name": "sku_extract_excerpts", "count": 2}], + "session_id": "session_logging", + } + response = httpx.Response( + 200, + json=response_body, + request=httpx.Request("POST", "https://api.parallel.ai/v1/extract"), + ) + + await PassThroughEndpointLogging().pass_through_async_success_handler( + httpx_response=response, + response_body=response_body, + logging_obj=logging_obj, + url_route="https://api.parallel.ai/v1/extract", + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + passthrough_logging_payload={ + "url": "https://api.parallel.ai/v1/extract", + "request_body": {"urls": ["https://example.com/1", "https://example.com/2"]}, + "request_method": "POST", + "cost_per_request": None, + }, + custom_llm_provider="parallel_ai", + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + dispatched_kwargs = logging_obj.dispatch_success_handlers.await_args.kwargs + assert dispatched_kwargs["model"] == PARALLEL_AI_EXTRACT_MODEL + assert dispatched_kwargs["custom_llm_provider"] == "parallel_ai" + assert dispatched_kwargs["response_cost"] == 0.002 + assert dispatched_kwargs["prefer_async_handlers"] is True From 8dce2171d666c8aed70566845679e594e919ab21 Mon Sep 17 00:00:00 2001 From: James Liounis Date: Fri, 21 Aug 2026 17:00:34 -0400 Subject: [PATCH 5/5] fix(parallel_ai): register the extract route with the gateway allowlist and types Three CI failures from adding the route: - test_gateway_plus_backend_covers_full_app requires every app route to be exposed by a component; /parallel_ai/ joins the pass-through prefixes. - The route appears in the proxy OpenAPI spec, so the dashboard types gain the path and its operation. - tests/.../parallel_ai/extract/ had no __init__.py while its parent package does, so pytest named the module by basename alone and it collided with fal_ai/test_cost_calculator.py. --- gateway/routes/allowlist.py | 1 + .../llms/parallel_ai/extract/__init__.py | 0 ui/litellm-dashboard/src/lib/http/schema.d.ts | 37 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 tests/test_litellm/llms/parallel_ai/extract/__init__.py diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 05baf98bbb5..b42c433c05c 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -94,6 +94,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/langfuse/", "/vllm/", "/mistral/", + "/parallel_ai/", "/groq/", "/voyage/", "/cursor/", diff --git a/tests/test_litellm/llms/parallel_ai/extract/__init__.py b/tests/test_litellm/llms/parallel_ai/extract/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f823577c336..b0f0e2ea20a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9842,6 +9842,23 @@ export interface paths { patch?: never; trace?: never; }; + "/parallel_ai/v1/extract": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Parallel Ai Extract Proxy Route */ + post: operations["parallel_ai_extract_proxy_route_parallel_ai_v1_extract_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/plugin-proxy/{plugin_name}/{path}": { parameters: { query?: never; @@ -49691,6 +49708,26 @@ export interface operations { }; }; }; + parallel_ai_extract_proxy_route_parallel_ai_v1_extract_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; plugin_proxy_plugin_proxy__plugin_name___path__get: { parameters: { query?: never;