diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 05baf98bbb5..b42c433c05c 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -94,6 +94,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/langfuse/", "/vllm/", "/mistral/", + "/parallel_ai/", "/groq/", "/voyage/", "/cursor/", diff --git a/litellm/__init__.py b/litellm/__init__.py index eebd2dad91e..d621feefc57 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1795,6 +1795,9 @@ if TYPE_CHECKING: from .llms.manus.responses.transformation import ( ManusResponsesAPIConfig as ManusResponsesAPIConfig, ) + from .llms.parallel_ai.responses.transformation import ( + ParallelAIResponsesConfig as ParallelAIResponsesConfig, + ) from .llms.perplexity.responses.transformation import ( PerplexityResponsesConfig as PerplexityResponsesConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 1c833256598..587cc73c654 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -239,6 +239,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMResponsesAPIConfig", "VolcEngineResponsesAPIConfig", "PerplexityResponsesConfig", + "ParallelAIResponsesConfig", "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", "BedrockMantleResponsesAPIConfig", @@ -967,6 +968,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.perplexity.responses.transformation", "PerplexityResponsesConfig", ), + "ParallelAIResponsesConfig": ( + ".llms.parallel_ai.responses.transformation", + "ParallelAIResponsesConfig", + ), "DatabricksResponsesAPIConfig": ( ".llms.databricks.responses.transformation", "DatabricksResponsesAPIConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 23e92d26a59..17f579b08ac 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -568,6 +568,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "ollama_chat", "deepinfra", "perplexity", + "parallel_ai", "mistral", "groq", "gigachat", @@ -747,6 +748,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES: Final = { openai_compatible_endpoints: Final[list] = [ "api.perplexity.ai", + "api.parallel.ai", "api.endpoints.anyscale.com/v1", "api.deepinfra.com/v1/openai", "api.mistral.ai/v1", @@ -809,6 +811,7 @@ openai_compatible_providers: Final[list] = [ "tencent", "deepinfra", "perplexity", + "parallel_ai", "xinference", "xai", "zai", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6536941a094..d0302924252 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2,7 +2,9 @@ ## File for 'response_cost' calculation in Logging import logging import time +from collections.abc import Mapping from functools import lru_cache +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast from httpx import Response @@ -181,6 +183,7 @@ _SEARCH_CALL_TYPES: Final = frozenset( ) _AREALTIME_CALL_TYPE: Final = CallTypes.arealtime.value +EMPTY_OPTIONAL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _MCP_CALL_TYPE: Final = CallTypes.call_mcp_tool.value @@ -655,6 +658,7 @@ def cost_per_token( if ( (model_info.get("input_cost_per_token") or 0.0) > 0 or (model_info.get("output_cost_per_token") or 0.0) > 0 + or (model_info.get("input_cost_per_request") or 0.0) > 0 or model_info.get("tiered_pricing") is not None ): return generic_cost_per_token( @@ -873,6 +877,20 @@ def _normalize_service_tier(service_tier: object) -> str | None: return service_tier +def _parallel_ai_response_pricing_model(model: str, optional_params: Mapping[str, object] | None) -> str | None: + from litellm.llms.parallel_ai.responses.cost_calculator import ( + is_parallel_ai_response_model, + parallel_ai_response_pricing_model, + ) + + if not is_parallel_ai_response_model(model): + return None + return parallel_ai_response_pricing_model( + model=model, + optional_params=optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS, + ) + + def _extract_service_tier(source: object) -> str | None: """Read a raw ``service_tier`` off a response body or usage object, dict or pydantic model alike.""" if isinstance(source, BaseModel): @@ -1231,12 +1249,25 @@ def completion_cost( service_tier = _normalize_service_tier(service_tier) + provider_for_cost: Final = _get_provider_for_cost_calc( + model=model, + custom_llm_provider=custom_llm_provider, + ) + pricing_base_model: Final = ( + _parallel_ai_response_pricing_model( + model=model or "parallel", + optional_params=optional_params, + ) + if base_model is None and custom_pricing is not True and provider_for_cost == LlmProviders.PARALLEL_AI.value + else base_model + ) + selected_model: Final = _select_model_name_for_cost_calc( model=model, completion_response=completion_response, custom_llm_provider=custom_llm_provider, custom_pricing=custom_pricing, - base_model=base_model, + base_model=pricing_base_model, router_model_id=router_model_id, ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 005e94ebe82..994c881b344 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -233,6 +233,14 @@ def get_llm_provider( if endpoint == "api.perplexity.ai": custom_llm_provider = "perplexity" dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY") + elif endpoint == "api.parallel.ai": + from litellm.llms.parallel_ai.common_utils import ( + resolve_parallel_ai_credentials, + ) + + custom_llm_provider = "parallel_ai" # rebind-ok: function returns custom_llm_provider + resolved = resolve_parallel_ai_credentials(api_base=api_base, api_key=api_key) + api_base, dynamic_api_key = resolved # rebind-ok: function returns api_base elif endpoint == "api.endpoints.anyscale.com/v1": custom_llm_provider = "anyscale" dynamic_api_key = get_secret_str("ANYSCALE_API_KEY") @@ -558,6 +566,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "parallel_ai": + from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials + + # parallel_ai serves the OpenAI Responses-compatible API at https://api.parallel.ai/v1/responses + api_base, dynamic_api_key = resolve_parallel_ai_credentials( # rebind-ok: function returns api_base + api_base=api_base, api_key=api_key + ) elif custom_llm_provider == "aiohttp_openai": return model, "aiohttp_openai", api_key, api_base elif custom_llm_provider == "anyscale": diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 0a52e1d283e..099b3a40e5a 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -918,6 +918,11 @@ def generic_cost_per_token( service_tier=service_tier, ) + ## FLAT PER-REQUEST COST + input_cost_per_request: Final = model_info.get("input_cost_per_request") + if input_cost_per_request: + prompt_cost += input_cost_per_request + ## CALCULATE OUTPUT COST text_tokens = 0 audio_tokens = 0 diff --git a/litellm/llms/parallel_ai/common_utils.py b/litellm/llms/parallel_ai/common_utils.py new file mode 100644 index 00000000000..56e213c76f0 --- /dev/null +++ b/litellm/llms/parallel_ai/common_utils.py @@ -0,0 +1,52 @@ +""" +Shared credential and endpoint resolution for the Parallel AI provider. +""" + +from typing import Final +from urllib.parse import urlsplit + +from litellm.secret_managers.main import get_secret_str + +PARALLEL_AI_API_BASE: Final = "https://api.parallel.ai" + + +def _origin(url: str) -> tuple[str, str]: + """Scheme and host of a base URL; a scheme-less base is read as https.""" + normalized: Final = url if "://" in url else f"https://{url}" + split: Final = urlsplit(normalized) + return split.scheme.lower(), split.netloc.lower() + + +def _is_trusted_api_base(caller_api_base: str, env_api_base: str | None) -> bool: + """Whether a caller-supplied base names an origin the server key may be sent to. + + Compares scheme as well as host: matching the host alone would accept + ``http://api.parallel.ai`` and put the server key on the wire in plaintext. + """ + trusted: Final = frozenset(_origin(base) for base in (PARALLEL_AI_API_BASE, env_api_base) if base) + scheme, host = _origin(caller_api_base) + return bool(host) and (scheme, host) in trusted + + +def resolve_parallel_ai_credentials(api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + """ + Resolve the effective (api_base, api_key) pair for a Parallel AI LLM request. + + A server-managed key (from env) is only used when the request targets the + provider default or the operator's own PARALLEL_AI_API_BASE override; a + caller-supplied base must bring its own key, so the server credential is + never forwarded to a caller-chosen host. + """ + env_api_base: Final = get_secret_str("PARALLEL_AI_API_BASE") + resolved_api_base: Final = api_base or env_api_base or PARALLEL_AI_API_BASE + + if api_key: + return resolved_api_base, api_key + + server_api_key: Final = get_secret_str("PARALLEL_AI_API_KEY") or get_secret_str("PARALLEL_API_KEY") + if server_api_key and api_base and not _is_trusted_api_base(api_base, env_api_base): + raise ValueError( + f"Refusing to send the server-configured Parallel AI key to the caller-supplied " + f"api_base '{api_base}'. Pass an explicit api_key when overriding api_base." + ) + return resolved_api_base, server_api_key diff --git a/litellm/llms/parallel_ai/extract/__init__.py b/litellm/llms/parallel_ai/extract/__init__.py new file mode 100644 index 00000000000..b3d6df3959b --- /dev/null +++ b/litellm/llms/parallel_ai/extract/__init__.py @@ -0,0 +1,6 @@ +from litellm.llms.parallel_ai.extract.cost_calculator import ( + PARALLEL_AI_EXTRACT_MODEL, + parallel_ai_extract_cost, +) + +__all__ = ["PARALLEL_AI_EXTRACT_MODEL", "parallel_ai_extract_cost"] diff --git a/litellm/llms/parallel_ai/extract/cost_calculator.py b/litellm/llms/parallel_ai/extract/cost_calculator.py new file mode 100644 index 00000000000..7ec090e2029 --- /dev/null +++ b/litellm/llms/parallel_ai/extract/cost_calculator.py @@ -0,0 +1,73 @@ +from typing import Annotated, Final + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr, ValidationError + +PARALLEL_AI_EXTRACT_COST_PER_URL: Final = 0.001 +PARALLEL_AI_EXTRACT_MODEL: Final = "parallel_ai/extract" +PARALLEL_AI_EXTRACT_USAGE_SKU: Final = "sku_extract_excerpts" + + +class _ParallelAIExtractUsageItem(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: StrictStr + count: Annotated[StrictInt, Field(ge=0)] + + +class _ParallelAIExtractUsageName(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: StrictStr + + +class _ParallelAIExtractBillingResponse(BaseModel): + model_config = ConfigDict(extra="ignore") + + usage: tuple[object, ...] | None = None + + +class _ParallelAIExtractBillingRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + urls: tuple[StrictStr, ...] = () + + +def _usage_url_count(response_body: object) -> int | None: + try: + parsed: Final = _ParallelAIExtractBillingResponse.model_validate(response_body) + except ValidationError: + return None + if parsed.usage is None: + return None + + target_items: Final = tuple(item for item in parsed.usage if _usage_name(item) == PARALLEL_AI_EXTRACT_USAGE_SKU) + if not target_items: + return 0 + + try: + usage_items: Final = tuple(_ParallelAIExtractUsageItem.model_validate(item) for item in target_items) + except ValidationError: + return None + return sum(item.count for item in usage_items) + + +def _usage_name(usage_item: object) -> str | None: + try: + parsed: Final = _ParallelAIExtractUsageName.model_validate(usage_item) + except ValidationError: + return None + return parsed.name + + +def _request_url_count(request_body: object) -> int: + try: + parsed: Final = _ParallelAIExtractBillingRequest.model_validate(request_body) + except ValidationError: + return 0 + return len(parsed.urls) + + +def parallel_ai_extract_cost(request_body: object, response_body: object) -> float: + usage_url_count: Final = _usage_url_count(response_body) + billed_url_count: Final = usage_url_count if usage_url_count is not None else _request_url_count(request_body) + return billed_url_count * PARALLEL_AI_EXTRACT_COST_PER_URL diff --git a/litellm/llms/parallel_ai/responses/__init__.py b/litellm/llms/parallel_ai/responses/__init__.py new file mode 100644 index 00000000000..fc0b2a4595d --- /dev/null +++ b/litellm/llms/parallel_ai/responses/__init__.py @@ -0,0 +1,7 @@ +""" +Parallel AI Responses API module. +""" + +from litellm.llms.parallel_ai.responses.transformation import ParallelAIResponsesConfig + +__all__ = ("ParallelAIResponsesConfig",) diff --git a/litellm/llms/parallel_ai/responses/cost_calculator.py b/litellm/llms/parallel_ai/responses/cost_calculator.py new file mode 100644 index 00000000000..8e10655b158 --- /dev/null +++ b/litellm/llms/parallel_ai/responses/cost_calculator.py @@ -0,0 +1,41 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +PARALLEL_AI_DEFAULT_REASONING_EFFORT: Final[str] = "medium" +PARALLEL_AI_EFFORT_TIER_MODELS: Final[Mapping[str, str]] = MappingProxyType( + { + "parallel-low": "low", + "parallel-medium": "medium", + "parallel-high": "high", + } +) +PARALLEL_AI_REASONING_EFFORTS: Final[frozenset[str]] = frozenset(PARALLEL_AI_EFFORT_TIER_MODELS.values()) +REASONING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def is_parallel_ai_response_model(model: str) -> bool: + model_without_provider: Final[str] = model.removeprefix("parallel_ai/") + return model_without_provider == "parallel" or model_without_provider in PARALLEL_AI_EFFORT_TIER_MODELS + + +def _reasoning_effort(optional_params: Mapping[str, object]) -> str: + try: + reasoning: Final = REASONING_ADAPTER.validate_python(optional_params.get("reasoning")) + except ValidationError: + return PARALLEL_AI_DEFAULT_REASONING_EFFORT + effort: Final[object] = reasoning.get("effort") + if isinstance(effort, str) and effort in PARALLEL_AI_REASONING_EFFORTS: + return effort + return PARALLEL_AI_DEFAULT_REASONING_EFFORT + + +def parallel_ai_response_pricing_model(model: str, optional_params: Mapping[str, object]) -> str: + """Return the cost-map model matching the effective Responses reasoning effort.""" + model_without_provider: Final[str] = model.removeprefix("parallel_ai/") + effort: Final[str] = PARALLEL_AI_EFFORT_TIER_MODELS.get(model_without_provider) or _reasoning_effort( + optional_params + ) + return f"parallel_ai/parallel-{effort}" diff --git a/litellm/llms/parallel_ai/responses/transformation.py b/litellm/llms/parallel_ai/responses/transformation.py new file mode 100644 index 00000000000..8fbc1bca322 --- /dev/null +++ b/litellm/llms/parallel_ai/responses/transformation.py @@ -0,0 +1,120 @@ +""" +Parallel AI Responses API, an OpenAI Responses-compatible web-research endpoint. + +Provider quirks: +- single `parallel` model; the performance tier is selected via `reasoning.effort` (low/medium/high), + or with the `parallel-low` / `parallel-medium` / `parallel-high` aliases, which pin the tier and + carry per-tier pricing in the cost map +- no `tools` param; web grounding is built in + +Ref: https://docs.parallel.ai/responses-api/responses-quickstart +""" + +from typing import Final + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials +from litellm.llms.parallel_ai.responses.cost_calculator import ( + PARALLEL_AI_EFFORT_TIER_MODELS as EFFORT_TIER_MODELS, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class ParallelAIResponsesConfig(OpenAIResponsesAPIConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseResponsesAPIConfig contract + """Ref: https://docs.parallel.ai/responses-api/responses-quickstart""" + return [ # mutable-ok: callers concatenate with other lists + "stream", + "reasoning", + "instructions", + "text", + "previous_response_id", + ] + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.PARALLEL_AI + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: BaseResponsesAPIConfig contract + """Parallel rejects unknown Responses params (e.g. `tools`), so unsupported keys are filtered out.""" + supported: Final = frozenset(self.get_supported_openai_params(model)) + return { # mutable-ok: request payload, merged downstream + key: value for key, value in response_api_optional_params.items() if key in supported + } + + def validate_environment( # mutable-ok: BaseResponsesAPIConfig contract + self, + headers: dict, # mutable-ok: BaseResponsesAPIConfig contract + model: str, + litellm_params: GenericLiteLLMParams | None, + ) -> dict: # mutable-ok: BaseResponsesAPIConfig contract + resolved_params: Final = litellm_params or GenericLiteLLMParams() + _, api_key = resolve_parallel_ai_credentials(api_base=resolved_params.api_base, api_key=resolved_params.api_key) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" # rebind-ok: stamping the caller's headers is the contract + return headers + + def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: # mutable-ok: base contract + resolved_api_base: Final = api_base or get_secret_str("PARALLEL_AI_API_BASE") or "https://api.parallel.ai" + trimmed: Final = resolved_api_base.rstrip("/") + if trimmed.endswith("/v1/responses"): + return trimmed + return f"{trimmed.removesuffix('/v1')}/v1/responses" + + def transform_responses_api_request( # mutable-ok: BaseResponsesAPIConfig contract + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict, # mutable-ok: BaseResponsesAPIConfig contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseResponsesAPIConfig contract + ) -> dict: # mutable-ok: BaseResponsesAPIConfig contract + """The tier aliases pin `reasoning.effort` so each alias bills at its cost map entry.""" + effort: Final = EFFORT_TIER_MODELS.get(model) + if effort is None: + return super().transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + return super().transform_responses_api_request( + model="parallel", + input=input, + response_api_optional_request_params={ # mutable-ok: request payload + **response_api_optional_request_params, + "reasoning": {"effort": effort}, # mutable-ok: request payload + }, + litellm_params=litellm_params, + headers=headers, + ) + + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """The API reports model `parallel` for every tier; cost tracking prices by response model, so the alias is restored.""" + response: Final = super().transform_response_api_response( + model=model, raw_response=raw_response, logging_obj=logging_obj + ) + if model in EFFORT_TIER_MODELS: + response.model = model + return response + + def supports_native_websocket(self) -> bool: + """Parallel AI does not support native WebSocket for the Responses API""" + return False diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9ca8d9e1bac..074c8d1fbc3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -35613,6 +35613,42 @@ "output_cost_per_token": 1.25e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, + "parallel_ai/parallel": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-high": { + "input_cost_per_request": 0.25, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-low": { + "input_cost_per_request": 0.01, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-medium": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, "parallel_ai/search": { "input_cost_per_query": 0.004, "litellm_provider": "parallel_ai", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed35691c6ec..0b49a14748b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -466,6 +466,7 @@ class LiteLLMRoutes(enum.Enum): "/openai_passthrough", "/assemblyai", "/eu.assemblyai", + "/parallel_ai", "/vllm", "/mistral", "/milvus", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 850b50227f4..1b71b3b229a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -26,6 +26,7 @@ from litellm.constants import ( BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks @@ -344,6 +345,52 @@ async def cohere_proxy_route( return received_value +def _parallel_ai_extract_url(api_base: str) -> str: + trimmed: Final = api_base.rstrip("/") + if trimmed.endswith("/v1/extract"): + return trimmed + return f"{trimmed.removesuffix('/v1')}/v1/extract" + + +@router.post( + "/parallel_ai/v1/extract", + tags=["Parallel AI Pass-through", "pass-through"], +) +async def parallel_ai_extract_proxy_route( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + deployment_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="parallel_ai", + region_name=None, + ) + api_base, api_key = resolve_parallel_ai_credentials( + api_base=None, + api_key=deployment_api_key, + ) + if api_key is None: + raise HTTPException( + status_code=500, + detail="PARALLEL_AI_API_KEY or PARALLEL_API_KEY is required for the Parallel AI Extract pass-through.", + ) + + endpoint_func: Final = create_pass_through_route( + endpoint="/parallel_ai/v1/extract", + target=_parallel_ai_extract_url(api_base), + custom_headers={ + "Content-Type": "application/json", + "x-api-key": api_key, + }, + custom_llm_provider="parallel_ai", + ) + return await endpoint_func( + request, + fastapi_response, + user_api_key_dict, + ) + + @router.api_route( "/vllm/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..1d98e647702 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/parallel_ai_passthrough_logging_handler.py @@ -0,0 +1,51 @@ +import json +from collections.abc import Mapping +from typing import Final +from urllib.parse import urlparse + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.parallel_ai.extract.cost_calculator import ( + PARALLEL_AI_EXTRACT_MODEL, + parallel_ai_extract_cost, +) +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.utils import StandardPassThroughResponseObject + + +class ParallelAIPassthroughLoggingHandler: + @staticmethod + def is_extract_route(url_route: str, custom_llm_provider: str | None) -> bool: + path: Final = urlparse(url_route).path.rstrip("/") + return custom_llm_provider == "parallel_ai" and path.endswith("/v1/extract") + + @staticmethod + def parallel_ai_extract_handler( + response_body: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + request_body: Mapping[str, object], + **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler + ) -> PassThroughEndpointLoggingTypedDict: + """ + Prices a Parallel AI Extract call from the URLs the provider reports as + billed (falling back to the requested URL count) and records model, + provider, and cost on the logging payload. + """ + response_cost: Final = parallel_ai_extract_cost( + request_body=request_body, + response_body=response_body, + ) + logging_obj.model_call_details.update( + model=PARALLEL_AI_EXTRACT_MODEL, + custom_llm_provider="parallel_ai", + response_cost=response_cost, + ) + + return { + "result": StandardPassThroughResponseObject(response=json.dumps(response_body)), + "kwargs": { # mutable-ok: the logging pipeline requires a plain kwargs dict + **kwargs, + "model": PARALLEL_AI_EXTRACT_MODEL, + "custom_llm_provider": "parallel_ai", + "response_cost": response_cost, + }, + } diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index c38566375f4..1553364bd4b 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -27,6 +27,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import ( from .llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) +from .llm_provider_handlers.parallel_ai_passthrough_logging_handler import ( + ParallelAIPassthroughLoggingHandler, +) from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) @@ -221,6 +224,16 @@ class PassThroughEndpointLogging: standard_logging_response_object = openai_passthrough_logging_handler_result["result"] kwargs = openai_passthrough_logging_handler_result["kwargs"] + elif ParallelAIPassthroughLoggingHandler.is_extract_route(url_route, custom_llm_provider): + parallel_ai_result = ParallelAIPassthroughLoggingHandler.parallel_ai_extract_handler( + response_body=response_body if isinstance(response_body, dict) else {}, + logging_obj=logging_obj, + request_body=request_body, + **kwargs, + ) + standard_logging_response_object = parallel_ai_result["result"] + kwargs = parallel_ai_result["kwargs"] # rebind-ok: every dispatch branch reassigns kwargs + elif self.is_cursor_route(url_route, custom_llm_provider): cursor_passthrough_logging_handler_result = CursorPassthroughLoggingHandler.cursor_passthrough_handler( httpx_response=httpx_response, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 73f46bd2181..3c0ad0e6d7c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -234,6 +234,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x input input_cost_per_character_above_128k_tokens: float | None # only for vertex ai models input_cost_per_query: float | None # only for rerank models + input_cost_per_request: ReadOnly[float | None] # flat per-request pricing input_cost_per_image: float | None # only for vertex ai models input_cost_per_image_token: float | None # for gpt-image-1 and similar models input_cost_per_video_token: float | None # for gemini omni models with video input @@ -3338,6 +3339,7 @@ class MirroredPricingParams(BaseModel): class CustomPricingLiteLLMParams(MirroredPricingParams): ## CUSTOM PRICING ## input_cost_per_second: float | None = None + input_cost_per_request: float | None = None output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None output_cost_per_second_480p: float | None = None @@ -3813,6 +3815,7 @@ class LlmProviders(str, Enum): CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" GDC = "gdc" + PARALLEL_AI = "parallel_ai" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 802dc151428..1e58e1c1aae 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5781,6 +5781,7 @@ def _get_model_info_helper( ), input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None), input_cost_per_query=_model_info.get("input_cost_per_query", None), + input_cost_per_request=_model_info.get("input_cost_per_request", None), input_cost_per_second=_model_info.get("input_cost_per_second", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), @@ -6444,6 +6445,11 @@ def validate_environment( keys_in_environment = True else: missing_keys.append("PERPLEXITYAI_API_KEY") + elif custom_llm_provider == "parallel_ai": + if "PARALLEL_AI_API_KEY" in os.environ or "PARALLEL_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("PARALLEL_AI_API_KEY") elif custom_llm_provider == "voyage": if "VOYAGE_API_KEY" in os.environ: keys_in_environment = True @@ -8579,6 +8585,8 @@ class ProviderConfigManager: return litellm.ManusResponsesAPIConfig() elif litellm.LlmProviders.PERPLEXITY == provider: return litellm.PerplexityResponsesConfig() + elif litellm.LlmProviders.PARALLEL_AI == provider: + return litellm.ParallelAIResponsesConfig() elif litellm.LlmProviders.DATABRICKS == provider: # Databricks Responses API is only compatible with OpenAI GPT models if model and "gpt" in model.lower(): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9ca8d9e1bac..074c8d1fbc3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -35613,6 +35613,42 @@ "output_cost_per_token": 1.25e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, + "parallel_ai/parallel": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-high": { + "input_cost_per_request": 0.25, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-low": { + "input_cost_per_request": 0.01, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, + "parallel_ai/parallel-medium": { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "mode": "responses", + "output_cost_per_token": 0.0, + "supports_reasoning": true, + "supports_response_schema": true + }, "parallel_ai/search": { "input_cost_per_query": 0.004, "litellm_provider": "parallel_ai", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d8d374c2c4..73cba4d8da6 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1934,9 +1934,9 @@ "display_name": "Parallel AI (`parallel_ai`)", "url": "https://docs.litellm.ai/docs/search/parallel_ai", "endpoints": { - "chat_completions": false, - "messages": false, - "responses": false, + "chat_completions": true, + "messages": true, + "responses": true, "embeddings": false, "image_generations": false, "audio_transcriptions": false, diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 97484851715..e9e24f787f0 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -36,7 +36,7 @@ "limit": 176 }, "B008": { - "limit": 503 + "limit": 505 }, "B009": { "limit": 58 diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 6f513ce1bd4..782f483bc44 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -3517,3 +3517,114 @@ def test_generic_cost_per_token_grok_46_long_context(_local_model_cost_map): ) assert prompt_cost == pytest.approx(200_000 * 4e-06 + 50_000 * 1e-06) assert completion_cost == pytest.approx(1_000 * 1.2e-05) + + +@pytest.fixture +def flat_per_request_model(): + """Register a synthetic model whose only price is a flat per-request rate.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + + model = "openai/flat-rate-test-model" + prev = litellm.model_cost.get(model) + litellm.model_cost[model] = { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openai", + "mode": "chat", + } + _invalidate_model_cost_lowercase_map() + try: + yield model + finally: + if prev is None: + litellm.model_cost.pop(model, None) + else: + litellm.model_cost[model] = prev + _invalidate_model_cost_lowercase_map() + + +def test_flat_per_request_cost_applies_once(flat_per_request_model): + """A flat per-request price bills the same regardless of token counts.""" + from litellm.types.utils import Usage + + small_usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + large_usage = Usage(prompt_tokens=10_000, completion_tokens=5_000, total_tokens=15_000) + + small_prompt_cost, small_completion_cost = generic_cost_per_token( + model=flat_per_request_model, + usage=small_usage, + custom_llm_provider="openai", + ) + large_prompt_cost, large_completion_cost = generic_cost_per_token( + model=flat_per_request_model, + usage=large_usage, + custom_llm_provider="openai", + ) + + assert small_prompt_cost == pytest.approx(0.05) + assert small_completion_cost == 0.0 + assert large_prompt_cost == pytest.approx(0.05) + assert large_completion_cost == 0.0 + + +def test_flat_per_request_cost_is_added_to_token_cost(): + """The flat rate stacks on top of token pricing rather than replacing it.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + from litellm.types.utils import Usage + + model = "openai/flat-plus-tokens-test-model" + litellm.model_cost[model] = { + "input_cost_per_request": 0.05, + "input_cost_per_token": 1e-05, + "output_cost_per_token": 2e-05, + "litellm_provider": "openai", + "mode": "chat", + } + _invalidate_model_cost_lowercase_map() + try: + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="openai", + ) + assert prompt_cost == pytest.approx(0.05 + 100 * 1e-05) + assert completion_cost == pytest.approx(50 * 2e-05) + finally: + litellm.model_cost.pop(model, None) + _invalidate_model_cost_lowercase_map() + + +def test_flat_per_request_cost_routes_through_cost_per_token(): + """The generic cost_per_token dispatch must not skip a model whose only + pricing is a flat per-request rate. + + Uses a provider with no dedicated cost_per_token branch, so the model + genuinely reaches the generic fallback whose guard is under test. + """ + from litellm.cost_calculator import cost_per_token + from litellm.types.utils import Usage + from litellm.utils import _invalidate_model_cost_lowercase_map + + model = "groq/flat-rate-only-test-model" + litellm.model_cost[model] = { + "input_cost_per_request": 0.05, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "groq", + "mode": "chat", + } + _invalidate_model_cost_lowercase_map() + try: + usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + prompt_cost, completion_cost = cost_per_token( + model=model, + custom_llm_provider="groq", + usage_object=usage, + ) + assert prompt_cost == pytest.approx(0.05) + assert completion_cost == 0.0 + finally: + litellm.model_cost.pop(model, None) + _invalidate_model_cost_lowercase_map() diff --git a/tests/test_litellm/llms/parallel_ai/extract/__init__.py b/tests/test_litellm/llms/parallel_ai/extract/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py b/tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py new file mode 100644 index 00000000000..83d8532da14 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/extract/test_cost_calculator.py @@ -0,0 +1,90 @@ +import pytest + +from litellm.llms.parallel_ai.extract.cost_calculator import parallel_ai_extract_cost + + +def test_extract_cost_prefers_provider_usage_over_requested_urls() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2", "https://example.com/3"]}, + response_body={"usage": [{"name": "sku_extract_excerpts", "count": 2}]}, + ) + + assert cost == pytest.approx(0.002) + + +def test_extract_cost_sums_repeated_usage_skus() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1"]}, + response_body={ + "usage": [ + {"name": "sku_extract_excerpts", "count": 1}, + {"name": "unrelated_sku", "count": 9}, + {"name": "sku_extract_excerpts", "count": 2}, + ] + }, + ) + + assert cost == pytest.approx(0.003) + + +def test_extract_cost_ignores_malformed_unrelated_usage_skus() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={ + "usage": [ + {"name": "sku_extract_excerpts", "count": 1}, + {"name": "unrelated_sku", "count": "not-an-integer"}, + ] + }, + ) + + assert cost == pytest.approx(0.001) + + +def test_extract_cost_treats_usage_without_extract_sku_as_unbilled() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"usage": [{"name": "unrelated_sku", "count": 2}]}, + ) + + assert cost == 0.0 + + +def test_extract_cost_treats_empty_usage_as_unbilled() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"usage": []}, + ) + + assert cost == 0.0 + + +def test_extract_cost_falls_back_to_requested_url_count_without_usage() -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"results": []}, + ) + + assert cost == pytest.approx(0.002) + + +@pytest.mark.parametrize("invalid_count", [True, -1, "2"]) +def test_extract_cost_falls_back_when_provider_usage_is_invalid(invalid_count: object) -> None: + cost = parallel_ai_extract_cost( + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + response_body={"usage": [{"name": "sku_extract_excerpts", "count": invalid_count}]}, + ) + + assert cost == pytest.approx(0.002) + + +@pytest.mark.parametrize( + "request_body", + [ + {}, + {"urls": "https://example.com"}, + {"urls": ["https://example.com", 42]}, + ], +) +def test_extract_cost_does_not_guess_from_invalid_request_urls(request_body: object) -> None: + assert parallel_ai_extract_cost(request_body=request_body, response_body={}) == 0.0 diff --git a/tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py b/tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py new file mode 100644 index 00000000000..cdbb017e512 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/responses/test_parallel_ai_responses_transformation.py @@ -0,0 +1,401 @@ +""" +Tests for Parallel AI Responses API transformation. + +Source: litellm/llms/parallel_ai/responses/transformation.py +""" + +import json +import pytest + +from litellm.llms.parallel_ai.responses.cost_calculator import parallel_ai_response_pricing_model +from litellm.llms.parallel_ai.responses.transformation import ParallelAIResponsesConfig +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +def _parallel_response_body(response_id: str = "resp_test") -> dict[str, object]: + return { + "id": response_id, + "object": "response", + "created_at": 1700000000, + "model": "parallel", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "grounded answer", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +class TestParallelAIResponsesConfig: + def test_provider_config_manager_returns_parallel_config(self): + config = ProviderConfigManager.get_provider_responses_api_config( + provider=LlmProviders.PARALLEL_AI, model="parallel" + ) + assert isinstance(config, ParallelAIResponsesConfig) + + @pytest.mark.parametrize( + "api_base,expected", + [ + (None, "https://api.parallel.ai/v1/responses"), + ("https://api.parallel.ai", "https://api.parallel.ai/v1/responses"), + ("https://api.parallel.ai/", "https://api.parallel.ai/v1/responses"), + ("https://api.parallel.ai/v1", "https://api.parallel.ai/v1/responses"), + ("https://proxy.example.com/v1/responses", "https://proxy.example.com/v1/responses"), + ], + ) + def test_get_complete_url(self, api_base, expected, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + config = ParallelAIResponsesConfig() + assert config.get_complete_url(api_base=api_base, litellm_params={}) == expected + + def test_validate_environment_sets_bearer_from_env(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + config = ParallelAIResponsesConfig() + + headers = config.validate_environment(headers={}, model="parallel", litellm_params=None) + assert headers["Authorization"] == "Bearer pk-test" + + def test_validate_environment_prefers_explicit_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + config = ParallelAIResponsesConfig() + + headers = config.validate_environment( + headers={}, + model="parallel", + litellm_params=GenericLiteLLMParams(api_key="pk-explicit"), + ) + assert headers["Authorization"] == "Bearer pk-explicit" + + def test_reasoning_effort_is_supported(self): + config = ParallelAIResponsesConfig() + supported = config.get_supported_openai_params(model="parallel") + assert "reasoning" in supported + assert "instructions" in supported + assert "previous_response_id" in supported + assert "tools" not in supported + + def test_unsupported_params_dropped_with_drop_params(self): + from litellm.responses.utils import ResponsesAPIRequestUtils + + config = ParallelAIResponsesConfig() + params = ResponsesAPIOptionalRequestParams( + reasoning={"effort": "high"}, + tools=[{"type": "web_search"}], + temperature=0.5, + ) + + mapped = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="parallel", + responses_api_provider_config=config, + response_api_optional_params=params, + drop_params=True, + ) + assert mapped["reasoning"] == {"effort": "high"} + assert "tools" not in mapped + assert "temperature" not in mapped + + def test_unsupported_params_raise_without_drop_params(self): + import litellm as litellm_module + from litellm.responses.utils import ResponsesAPIRequestUtils + + config = ParallelAIResponsesConfig() + params = ResponsesAPIOptionalRequestParams(tools=[{"type": "web_search"}]) + + with pytest.raises(litellm_module.UnsupportedParamsError): + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="parallel", + responses_api_provider_config=config, + response_api_optional_params=params, + drop_params=False, + ) + + def test_transform_request_sends_parallel_model(self): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model="parallel", + input="What is the latest AI news?", + response_api_optional_request_params={"reasoning": {"effort": "low"}}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["model"] == "parallel" + assert request["input"] == "What is the latest AI news?" + assert request["reasoning"] == {"effort": "low"} + + def test_no_native_websocket(self): + assert ParallelAIResponsesConfig().supports_native_websocket() is False + + def test_untrusted_api_base_refuses_server_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + config = ParallelAIResponsesConfig() + + with pytest.raises(ValueError, match="Refusing to send"): + config.validate_environment( + headers={}, + model="parallel", + litellm_params=GenericLiteLLMParams(api_base="https://attacker.example.com"), + ) + + def test_untrusted_api_base_with_explicit_key_is_allowed(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + config = ParallelAIResponsesConfig() + + headers = config.validate_environment( + headers={}, + model="parallel", + litellm_params=GenericLiteLLMParams(api_base="https://proxy.example.com", api_key="pk-caller"), + ) + assert headers["Authorization"] == "Bearer pk-caller" + + +class TestParallelAICompletionBridge: + @pytest.mark.respx() + def test_completion_routes_through_responses_api(self, respx_mock, monkeypatch): + """litellm.completion() on a mode=responses model must bridge to /v1/responses.""" + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + import litellm as litellm_module + + litellm_module.model_cost = litellm_module.get_model_cost_map(url="") + + respx_mock.post("https://api.parallel.ai/v1/responses").respond(json=_parallel_response_body()) + + response = litellm_module.completion( + model="parallel_ai/parallel", + messages=[{"role": "user", "content": "question"}], + ) + + assert response.choices[0].message.content == "grounded answer" + + +class TestParallelAIEffortTierAliases: + @pytest.mark.parametrize( + "alias,effort", + [("parallel-low", "low"), ("parallel-medium", "medium"), ("parallel-high", "high")], + ) + def test_alias_pins_model_and_effort(self, alias, effort): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model=alias, + input="question", + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["model"] == "parallel" + assert request["reasoning"] == {"effort": effort} + + def test_alias_effort_wins_over_explicit_reasoning(self): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model="parallel-high", + input="question", + response_api_optional_request_params={"reasoning": {"effort": "low"}}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["reasoning"] == {"effort": "high"} + + def test_plain_model_passes_reasoning_through(self): + config = ParallelAIResponsesConfig() + request = config.transform_responses_api_request( + model="parallel", + input="question", + response_api_optional_request_params={"reasoning": {"effort": "low"}}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["model"] == "parallel" + assert request["reasoning"] == {"effort": "low"} + + def test_alias_restored_on_response_model_for_cost_tracking(self): + from unittest.mock import MagicMock + + config = ParallelAIResponsesConfig() + raw = MagicMock() + raw.json.return_value = { + "id": "resp_1", + "object": "response", + "created_at": 1700000000, + "model": "parallel", + "status": "completed", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + response = config.transform_response_api_response( + model="parallel-low", raw_response=raw, logging_obj=MagicMock() + ) + assert response.model == "parallel-low" + + raw.json.return_value["model"] = "parallel" + response_plain = config.transform_response_api_response( + model="parallel", raw_response=raw, logging_obj=MagicMock() + ) + assert response_plain.model == "parallel" + + +class TestParallelAIReasoningEffortPricing: + @pytest.mark.parametrize( + "model,optional_params,expected_model", + [ + ("parallel", {"reasoning": {"effort": "low"}}, "parallel_ai/parallel-low"), + ("parallel", {}, "parallel_ai/parallel-medium"), + ("parallel", {"reasoning": {"effort": "high"}}, "parallel_ai/parallel-high"), + ("parallel_ai/parallel", {"reasoning": {"effort": "low"}}, "parallel_ai/parallel-low"), + ("parallel-high", {"reasoning": {"effort": "low"}}, "parallel_ai/parallel-high"), + ], + ) + def test_pricing_model_uses_effective_reasoning_effort(self, model, optional_params, expected_model): + assert parallel_ai_response_pricing_model(model=model, optional_params=optional_params) == expected_model + + @pytest.mark.parametrize( + "model,optional_params,expected_cost", + [ + ("parallel_ai/parallel", {"reasoning": {"effort": "low"}}, 0.01), + ("parallel_ai/parallel", {}, 0.05), + ("parallel_ai/parallel", {"reasoning": {"effort": "high"}}, 0.25), + ("parallel_ai/parallel-high", {"reasoning": {"effort": "low"}}, 0.25), + ], + ) + def test_completion_cost_uses_effective_reasoning_effort(self, model, optional_params, expected_cost, monkeypatch): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + response = ResponsesAPIResponse( + id="resp_cost", + created_at=1700000000, + model="parallel", + output=[], + usage={"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + ) + + cost = litellm_module.completion_cost( + completion_response=response, + model=model, + optional_params=optional_params, + call_type="responses", + ) + + assert cost == pytest.approx(expected_cost) + + def test_explicit_base_model_remains_authoritative(self, monkeypatch): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + response = ResponsesAPIResponse( + id="resp_custom_base", + created_at=1700000000, + model="parallel", + output=[], + usage={"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + ) + + cost = litellm_module.completion_cost( + completion_response=response, + model="parallel_ai/parallel", + optional_params={"reasoning": {"effort": "high"}}, + base_model="parallel_ai/parallel-low", + call_type="responses", + ) + + assert cost == pytest.approx(0.01) + + @pytest.mark.respx() + @pytest.mark.parametrize( + "reasoning,expected_cost", + [ + ({"effort": "low"}, 0.01), + (None, 0.05), + ({"effort": "high"}, 0.25), + ], + ) + def test_responses_records_effort_aware_cost(self, reasoning, expected_cost, respx_mock, monkeypatch): + import litellm as litellm_module + + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + route = respx_mock.post("https://api.parallel.ai/v1/responses").respond(json=_parallel_response_body()) + + response = litellm_module.responses( + model="parallel_ai/parallel", + input="question", + reasoning=reasoning, + ) + + assert response.model == "parallel" + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) + request_body = json.loads(route.calls.last.request.content) + if reasoning is None: + assert "reasoning" not in request_body + else: + assert request_body["reasoning"] == reasoning + + @pytest.mark.respx() + def test_streaming_response_records_high_effort_cost(self, respx_mock, monkeypatch): + import litellm as litellm_module + + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + monkeypatch.setattr(litellm_module, "include_cost_in_streaming_usage", True) + completed_event = { + "type": "response.completed", + "response": _parallel_response_body(response_id="resp_stream"), + } + stream_body = f"data: {json.dumps(completed_event)}\n\ndata: [DONE]\n\n".encode() + respx_mock.post("https://api.parallel.ai/v1/responses").respond( + content=stream_body, + headers={"content-type": "text/event-stream"}, + ) + + stream = litellm_module.responses( + model="parallel_ai/parallel", + input="question", + reasoning={"effort": "high"}, + stream=True, + ) + events = list(stream) + + assert len(events) == 1 + assert events[0].response.model == "parallel" + assert events[0].response.usage.cost == pytest.approx(0.25) + + @pytest.mark.respx() + def test_completion_bridge_records_high_effort_cost(self, respx_mock, monkeypatch): + import litellm as litellm_module + + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + monkeypatch.setattr(litellm_module, "model_cost", litellm_module.get_model_cost_map(url="")) + route = respx_mock.post("https://api.parallel.ai/v1/responses").respond(json=_parallel_response_body()) + + response = litellm_module.completion( + model="parallel_ai/parallel", + messages=[{"role": "user", "content": "question"}], + reasoning={"effort": "high"}, + ) + + assert response.choices[0].message.content == "grounded answer" + assert response._hidden_params["response_cost"] == pytest.approx(0.25) + request_body = json.loads(route.calls.last.request.content) + assert request_body["reasoning"] == {"effort": "high"} diff --git a/tests/test_litellm/llms/parallel_ai/test_common_utils.py b/tests/test_litellm/llms/parallel_ai/test_common_utils.py new file mode 100644 index 00000000000..e70ea8b557a --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/test_common_utils.py @@ -0,0 +1,123 @@ +""" +Tests for Parallel AI credential and provider resolution. + +Source: litellm/llms/parallel_ai/common_utils.py +""" + +import pytest + +import litellm +from litellm.llms.parallel_ai.common_utils import resolve_parallel_ai_credentials + + +class TestResolveParallelAICredentials: + @pytest.fixture(autouse=True) + def _clean_env(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + + def test_defaults(self, monkeypatch): + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + + api_base, api_key = resolve_parallel_ai_credentials(api_base=None, api_key=None) + assert api_base == "https://api.parallel.ai" + assert api_key == "pk-test" + + def test_prefers_parallel_ai_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-primary") + monkeypatch.setenv("PARALLEL_API_KEY", "pk-fallback") + + _, api_key = resolve_parallel_ai_credentials(api_base=None, api_key=None) + assert api_key == "pk-primary" + + def test_plaintext_provider_host_is_not_trusted(self, monkeypatch): + """A host-only trust check would send the server key over plaintext HTTP.""" + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + with pytest.raises(ValueError, match="Refusing to send"): + resolve_parallel_ai_credentials(api_base="http://api.parallel.ai", api_key=None) + + def test_https_provider_host_is_trusted(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://api.parallel.ai", api_key=None) + assert api_base == "https://api.parallel.ai" + assert api_key == "pk-env" + + def test_operator_plaintext_override_is_trusted(self, monkeypatch): + """The operator's own PARALLEL_AI_API_BASE is trusted at the scheme it names.""" + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + monkeypatch.setenv("PARALLEL_AI_API_BASE", "http://parallel-proxy.internal") + + _, api_key = resolve_parallel_ai_credentials(api_base="http://parallel-proxy.internal", api_key=None) + assert api_key == "pk-env" + + def test_explicit_args_win(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://proxy.example.com", api_key="pk-explicit") + assert api_base == "https://proxy.example.com" + assert api_key == "pk-explicit" + + def test_untrusted_api_base_refuses_server_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + + with pytest.raises(ValueError, match="Refusing to send"): + resolve_parallel_ai_credentials(api_base="https://attacker.example.com", api_key=None) + + def test_operator_env_base_is_trusted(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-server-secret") + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://proxy.internal.example.com") + + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://proxy.internal.example.com", api_key=None) + assert api_base == "https://proxy.internal.example.com" + assert api_key == "pk-server-secret" + + def test_untrusted_base_without_server_key_returns_none(self): + api_base, api_key = resolve_parallel_ai_credentials(api_base="https://proxy.example.com", api_key=None) + assert api_base == "https://proxy.example.com" + assert api_key is None + + +class TestParallelAIProviderWiring: + def test_get_llm_provider_routes_parallel_ai(self, monkeypatch): + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + model, provider, api_key, api_base = litellm.get_llm_provider("parallel_ai/parallel") + assert model == "parallel" + assert provider == "parallel_ai" + assert api_key == "pk-test" + assert api_base == "https://api.parallel.ai" + + def test_get_llm_provider_detects_parallel_api_base(self, monkeypatch): + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + + model, provider, api_key, api_base = litellm.get_llm_provider( + model="parallel", api_base="https://api.parallel.ai" + ) + assert provider == "parallel_ai" + assert api_key == "pk-test" + + def test_validate_environment_reports_missing_key(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + + result = litellm.validate_environment(model="parallel_ai/parallel") + assert result["keys_in_environment"] is False + assert "PARALLEL_AI_API_KEY" in result["missing_keys"] + + def test_validate_environment_accepts_either_key_name(self, monkeypatch): + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "pk-test") + + result = litellm.validate_environment(model="parallel_ai/parallel") + assert result["keys_in_environment"] is True + + def test_api_base_detection_keeps_explicit_caller_key(self, monkeypatch): + monkeypatch.setenv("PARALLEL_AI_API_KEY", "pk-env") + + model, provider, api_key, api_base = litellm.get_llm_provider( + model="parallel", api_base="https://api.parallel.ai", api_key="pk-explicit" + ) + assert provider == "parallel_ai" + assert api_key == "pk-explicit" diff --git a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py new file mode 100644 index 00000000000..4df5f3d5333 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_extract_gateway.py @@ -0,0 +1,185 @@ +"""Gateway coverage for Parallel AI's Extract pass-through.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from typing import Final + +import pytest +from fastapi.testclient import TestClient + +import litellm +from litellm import Router +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +PARALLEL_EXTRACT_URL: Final = "https://api.parallel.ai/v1/extract" + + +@pytest.fixture +def client() -> TestClient: + return TestClient(proxy_server.app, raise_server_exceptions=False) + + +@pytest.fixture +def auth_as() -> Iterator[None]: + async def _authorized_request() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-sk-test", + user_id="parallel-test-user", + ) + + previous: Final = proxy_server.app.dependency_overrides.get(user_api_key_auth) + proxy_server.app.dependency_overrides[user_api_key_auth] = _authorized_request + try: + yield + finally: + if previous is None: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + else: + proxy_server.app.dependency_overrides[user_api_key_auth] = previous + + +def _parallel_extract_body() -> dict[str, object]: + return { + "extract_id": "extract_parallel_gateway", + "results": [ + { + "url": "https://example.com/parallel", + "title": "Parallel result", + "publish_date": "2026-08-14", + "excerpts": ["Focused excerpt"], + "full_content": "# Full content", + } + ], + "errors": [ + { + "url": "https://example.com/unavailable", + "error_type": "fetch_error", + "http_status_code": 503, + "content": "Upstream unavailable", + } + ], + "warnings": None, + "usage": [{"name": "sku_extract_excerpts", "count": 2}], + "session_id": "session_parallel_gateway", + } + + +def _parallel_router() -> Router: + return Router( + model_list=[ + { + "model_name": "parallel-gateway", + "litellm_params": { + "model": "parallel_ai/parallel", + "api_key": "parallel-responses-key", + "use_in_pass_through": True, + }, + } + ], + num_retries=0, + ) + + +def test_parallel_extract_gateway_route(client, auth_as, monkeypatch, respx_mock): + """The native Extract route preserves Parallel's V1 request and partial-success response.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", _parallel_router()) + upstream_route = respx_mock.post(PARALLEL_EXTRACT_URL).respond(json=_parallel_extract_body()) + request_body = { + "urls": [ + "https://example.com/parallel", + "https://example.com/unavailable", + ], + "objective": "Find the integration details", + "search_queries": ["Parallel integration"], + "max_chars_total": 50000, + "session_id": "session_parallel_gateway", + "client_model": "gpt-5.4", + "advanced_settings": { + "fetch_policy": { + "max_age_seconds": 3600, + "timeout_seconds": 30, + "disable_cache_fallback": False, + }, + "excerpt_settings": {"max_chars_per_result": 5000}, + "full_content": {"max_chars_per_result": 50000}, + }, + } + + response = client.post("/parallel_ai/v1/extract", json=request_body) + + assert response.status_code == 200, response.text + assert response.json() == _parallel_extract_body() + assert upstream_route.called + + upstream_request = upstream_route.calls.last.request + assert upstream_request.headers["x-api-key"] == "parallel-responses-key" + assert "authorization" not in upstream_request.headers + assert json.loads(upstream_request.content) == request_body + + +def test_parallel_extract_route_is_classified_as_an_llm_api_route() -> None: + assert RouteChecks.is_llm_api_route(route="/parallel_ai/v1/extract") is True + + +def test_parallel_extract_gateway_uses_environment_configuration(client, auth_as, monkeypatch, respx_mock): + custom_url = "https://parallel-proxy.example.com/v1/extract" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.setenv("PARALLEL_API_KEY", "parallel-env-key") + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://parallel-proxy.example.com/v1") + upstream_route = respx_mock.post(custom_url).respond(json=_parallel_extract_body()) + + response = client.post( + "/parallel_ai/v1/extract", + json={"urls": ["https://example.com/parallel"]}, + ) + + assert response.status_code == 200, response.text + assert upstream_route.called + assert upstream_route.calls.last.request.headers["x-api-key"] == "parallel-env-key" + + +def test_parallel_extract_gateway_requires_a_provider_key(client, auth_as, monkeypatch) -> None: + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + + response = client.post( + "/parallel_ai/v1/extract", + json={"urls": ["https://example.com/parallel"]}, + ) + + assert response.status_code == 500 + assert response.json()["detail"] == ( + "PARALLEL_AI_API_KEY or PARALLEL_API_KEY is required for the Parallel AI Extract pass-through." + ) + + +def test_parallel_extract_gateway_preserves_validation_errors(client, auth_as, monkeypatch, respx_mock): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", _parallel_router()) + error_body = { + "error": { + "type": "validation_error", + "message": "urls must contain at most 20 items", + } + } + respx_mock.post(PARALLEL_EXTRACT_URL).respond(status_code=422, json=error_body) + + response = client.post( + "/parallel_ai/v1/extract", + json={"urls": [f"https://example.com/{index}" for index in range(21)]}, + ) + + assert response.status_code == 422 + assert response.json() == error_body diff --git a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py new file mode 100644 index 00000000000..30f93e85676 --- /dev/null +++ b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_responses_gateway.py @@ -0,0 +1,138 @@ +"""Gateway coverage for Parallel AI's Responses integration.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Final +from unittest.mock import AsyncMock + +import httpx +import pytest +from fastapi.testclient import TestClient + +import litellm +from litellm import Router +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +PARALLEL_RESPONSES_URL: Final = "https://api.parallel.ai/v1/responses" + + +@pytest.fixture +def client() -> TestClient: + return TestClient(proxy_server.app, raise_server_exceptions=False) + + +@pytest.fixture +def auth_as() -> Iterator[None]: + async def _authorized_request() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-sk-test", + user_id="parallel-test-user", + ) + + previous: Final = proxy_server.app.dependency_overrides.get(user_api_key_auth) + proxy_server.app.dependency_overrides[user_api_key_auth] = _authorized_request + try: + yield + finally: + if previous is None: + proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) + else: + proxy_server.app.dependency_overrides[user_api_key_auth] = previous + + +def _parallel_response_body() -> dict[str, object]: + return { + "id": "resp_parallel_gateway", + "object": "response", + "created_at": 1700000000, + "model": "parallel", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_parallel_gateway", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Parallel grounded answer", + "annotations": [], + } + ], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +def _parallel_router() -> Router: + return Router( + model_list=[ + { + "model_name": "parallel-gateway", + "litellm_params": { + "model": "parallel_ai/parallel", + "api_key": "parallel-responses-key", + "use_in_pass_through": True, + }, + } + ], + num_retries=0, + ) + + +def _mock_async_post( + monkeypatch, + *, + url: str, + response_body: dict[str, object], +) -> AsyncMock: + response = httpx.Response( + status_code=200, + json=response_body, + request=httpx.Request("POST", url), + ) + mock_post = AsyncMock(return_value=response) + monkeypatch.setattr(AsyncHTTPHandler, "post", mock_post) + return mock_post + + +def test_parallel_responses_gateway_route(client, auth_as, monkeypatch): + """The primary LLM route reaches Parallel's native Responses endpoint.""" + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setattr(proxy_server, "llm_router", _parallel_router()) + mock_post = _mock_async_post( + monkeypatch, + url=PARALLEL_RESPONSES_URL, + response_body=_parallel_response_body(), + ) + + response = client.post( + "/v1/responses", + json={"model": "parallel-gateway", "input": "Research Parallel"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["output"][0]["content"][0]["text"] == "Parallel grounded answer" + + request_kwargs = mock_post.await_args.kwargs + assert request_kwargs["url"] == PARALLEL_RESPONSES_URL + assert request_kwargs["headers"]["Authorization"] == "Bearer parallel-responses-key" + assert request_kwargs["json"] == { + "model": "parallel", + "input": "Research Parallel", + } diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..431ab51b181 --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_parallel_ai_passthrough_logging_handler.py @@ -0,0 +1,133 @@ +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.llms.parallel_ai.extract.cost_calculator import PARALLEL_AI_EXTRACT_MODEL +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.parallel_ai_passthrough_logging_handler import ( + ParallelAIPassthroughLoggingHandler, +) +from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging + + +def test_extract_route_detection_requires_parallel_provider() -> None: + assert ParallelAIPassthroughLoggingHandler.is_extract_route( + "https://api.parallel.ai/v1/extract", + "parallel_ai", + ) + assert not ParallelAIPassthroughLoggingHandler.is_extract_route( + "https://api.parallel.ai/v1/extract", + None, + ) + assert not ParallelAIPassthroughLoggingHandler.is_extract_route( + "https://api.parallel.ai/v1/search", + "parallel_ai", + ) + + +def test_extract_handler_sets_usage_aware_cost_and_model() -> None: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "parallel-extract-call" + logging_obj.model_call_details = {} + response_body = { + "extract_id": "extract_test", + "results": [], + "errors": [], + "usage": [{"name": "sku_extract_excerpts", "count": 2}], + "session_id": "session_test", + } + + result = ParallelAIPassthroughLoggingHandler.parallel_ai_extract_handler( + response_body=response_body, + logging_obj=logging_obj, + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + ) + + assert result["kwargs"]["model"] == PARALLEL_AI_EXTRACT_MODEL + assert result["kwargs"]["custom_llm_provider"] == "parallel_ai" + assert result["kwargs"]["response_cost"] == 0.002 + assert logging_obj.model_call_details["model"] == PARALLEL_AI_EXTRACT_MODEL + assert logging_obj.model_call_details["custom_llm_provider"] == "parallel_ai" + assert logging_obj.model_call_details["response_cost"] == 0.002 + + +def test_success_handler_dispatches_parallel_extract_billing() -> None: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "parallel-extract-dispatch" + logging_obj.model_call_details = {} + response_body = { + "extract_id": "extract_dispatch", + "results": [], + "errors": [], + "usage": [{"name": "sku_extract_excerpts", "count": 1}], + "session_id": "session_dispatch", + } + response = httpx.Response( + 200, + json=response_body, + request=httpx.Request("POST", "https://api.parallel.ai/v1/extract"), + ) + + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=response, + response_body=response_body, + request_body={"urls": ["https://example.com"]}, + logging_obj=logging_obj, + url_route="https://api.parallel.ai/v1/extract", + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="parallel_ai", + ) + + assert normalized["standard_logging_response_object"] is not None + assert normalized["kwargs"]["model"] == PARALLEL_AI_EXTRACT_MODEL + assert normalized["kwargs"]["response_cost"] == 0.001 + + +@pytest.mark.asyncio +async def test_success_handler_sends_extract_cost_to_async_loggers() -> None: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "parallel-extract-logging" + logging_obj.model_call_details = {} + logging_obj.dispatch_success_handlers = AsyncMock() + response_body = { + "extract_id": "extract_logging", + "results": [], + "errors": [], + "usage": [{"name": "sku_extract_excerpts", "count": 2}], + "session_id": "session_logging", + } + response = httpx.Response( + 200, + json=response_body, + request=httpx.Request("POST", "https://api.parallel.ai/v1/extract"), + ) + + await PassThroughEndpointLogging().pass_through_async_success_handler( + httpx_response=response, + response_body=response_body, + logging_obj=logging_obj, + url_route="https://api.parallel.ai/v1/extract", + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"urls": ["https://example.com/1", "https://example.com/2"]}, + passthrough_logging_payload={ + "url": "https://api.parallel.ai/v1/extract", + "request_body": {"urls": ["https://example.com/1", "https://example.com/2"]}, + "request_method": "POST", + "cost_per_request": None, + }, + custom_llm_provider="parallel_ai", + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + dispatched_kwargs = logging_obj.dispatch_success_handlers.await_args.kwargs + assert dispatched_kwargs["model"] == PARALLEL_AI_EXTRACT_MODEL + assert dispatched_kwargs["custom_llm_provider"] == "parallel_ai" + assert dispatched_kwargs["response_cost"] == 0.002 + assert dispatched_kwargs["prefer_async_handlers"] is True diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e0b8cf19159..4c46062e004 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9847,6 +9847,23 @@ export interface paths { patch?: never; trace?: never; }; + "/parallel_ai/v1/extract": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Parallel Ai Extract Proxy Route */ + post: operations["parallel_ai_extract_proxy_route_parallel_ai_v1_extract_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/plugin-proxy/{plugin_name}/{path}": { parameters: { query?: never; @@ -27572,6 +27589,8 @@ export interface components { input_cost_per_pixel?: number | null; /** Input Cost Per Query */ input_cost_per_query?: number | null; + /** Input Cost Per Request */ + input_cost_per_request?: number | null; /** Input Cost Per Second */ input_cost_per_second?: number | null; /** Input Cost Per Token */ @@ -36792,6 +36811,8 @@ export interface components { input_cost_per_pixel?: number | null; /** Input Cost Per Query */ input_cost_per_query?: number | null; + /** Input Cost Per Request */ + input_cost_per_request?: number | null; /** Input Cost Per Second */ input_cost_per_second?: number | null; /** Input Cost Per Token */ @@ -49755,6 +49776,26 @@ export interface operations { }; }; }; + parallel_ai_extract_proxy_route_parallel_ai_v1_extract_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; plugin_proxy_plugin_proxy__plugin_name___path__get: { parameters: { query?: never;