From 0c27f206925d1790e35c84086832a314731471d7 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 17 Feb 2026 16:15:00 -0300 Subject: [PATCH] feat(openai_like): add Responses API support to JSON provider system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add infrastructure for JSON-declared providers to support /v1/responses via `supported_endpoints` field in providers.json. Simplify Perplexity responses config from 410 to 40 lines by moving cost dict→float parsing to generic validators in ResponseAPIUsage and Usage. - Add `supported_endpoints` field to SimpleProviderConfig (default: []) - Add `supports_responses_api()` to JSONProviderRegistry - Create OpenAILikeResponsesConfig base class for responses API - Add `create_responses_config_class()` with class caching - ProviderConfigManager: Python classes take priority over JSON fallback - Fix ResponseAPIUsage.cost field_validator to handle dict cost objects - Fix Usage.__init__ to handle dict cost from chat completions - Simplify PerplexityResponsesConfig with get_supported_openai_params guard - Add 20 unit tests including Python-over-JSON priority test --- .../adding_openai_compatible_providers.md | 42 +- litellm/llms/openai_like/README.md | 38 +- litellm/llms/openai_like/dynamic_config.py | 60 +++ litellm/llms/openai_like/json_loader.py | 9 + litellm/llms/openai_like/providers.json | 5 + .../llms/openai_like/responses/__init__.py | 5 + .../openai_like/responses/transformation.py | 51 +++ .../perplexity/responses/transformation.py | 395 ++---------------- litellm/responses/main.py | 12 +- litellm/types/llms/openai.py | 8 + litellm/types/utils.py | 8 +- litellm/utils.py | 45 +- .../llms/openai_like/responses/__init__.py | 0 .../responses/test_openai_like_responses.py | 337 +++++++++++++++ 14 files changed, 639 insertions(+), 376 deletions(-) create mode 100644 litellm/llms/openai_like/responses/__init__.py create mode 100644 litellm/llms/openai_like/responses/transformation.py create mode 100644 tests/test_litellm/llms/openai_like/responses/__init__.py create mode 100644 tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py diff --git a/docs/my-website/docs/contributing/adding_openai_compatible_providers.md b/docs/my-website/docs/contributing/adding_openai_compatible_providers.md index bb89eea35bf..598d3dfe89a 100644 --- a/docs/my-website/docs/contributing/adding_openai_compatible_providers.md +++ b/docs/my-website/docs/contributing/adding_openai_compatible_providers.md @@ -80,6 +80,36 @@ That's it! The provider is now available. } ``` +## Responses API Support + +If your provider also supports the OpenAI Responses API (`/v1/responses`), add `supported_endpoints`: + +```json +{ + "your_provider": { + "base_url": "https://api.yourprovider.com/v1", + "api_key_env": "YOUR_PROVIDER_API_KEY", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + } +} +``` + +This enables `litellm.responses()` with zero additional code: + +```python +import litellm + +response = litellm.responses( + model="your_provider/model-name", + input="Hello, what can you do?", +) +print(response.output) +``` + +If `supported_endpoints` is omitted, it defaults to `[]`. Chat completions is always enabled for JSON providers regardless of this field. + +The provider inherits all request/response handling from OpenAI's Responses API — streaming, tools, and all standard parameters work out of the box. + ## Usage ```python @@ -89,11 +119,17 @@ import os # Set your API key os.environ["YOUR_PROVIDER_API_KEY"] = "your-key-here" -# Use the provider +# Chat completions response = litellm.completion( model="your_provider/model-name", messages=[{"role": "user", "content": "Hello"}], ) + +# Responses API (if supported_endpoints includes "/v1/responses") +response = litellm.responses( + model="your_provider/model-name", + input="Hello", +) ``` ## When to Use Python Instead @@ -105,7 +141,9 @@ Use a Python config class if you need: - Provider-specific streaming logic - Advanced tool calling modifications -For these cases, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`. +For chat completions, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`. + +For responses API with small overrides, inherit from `OpenAIResponsesAPIConfig` and override only what's needed. See `litellm/llms/perplexity/responses/transformation.py` for a minimal example (~40 lines vs 400+). ## Testing diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index 2e7a32f65a7..e9aaafe48a1 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -10,8 +10,9 @@ Instead of creating a full Python module for simple OpenAI-compatible providers, - `providers.json` - Configuration file for all JSON-based providers - `json_loader.py` - Loads and parses the JSON configuration -- `dynamic_config.py` - Generates Python config classes from JSON -- `chat/` - Existing OpenAI-like chat completion handlers +- `dynamic_config.py` - Generates Python config classes from JSON (chat + responses) +- `chat/` - OpenAI-like chat completion handlers +- `responses/` - OpenAI-like Responses API handlers ## Adding a New Provider @@ -96,6 +97,32 @@ response = litellm.completion( ) ``` +## Responses API Support + +Providers that support the OpenAI Responses API (`/v1/responses`) can declare it via `supported_endpoints`: + +```json +{ + "your_provider": { + "base_url": "https://api.yourprovider.com/v1", + "api_key_env": "YOUR_PROVIDER_API_KEY", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + } +} +``` + +This enables `litellm.responses(model="your_provider/model-name", ...)` with zero Python code. +The provider inherits all request/response handling from OpenAI's Responses API config. + +If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers). + +### How It Works + +1. `json_loader.py` checks `supported_endpoints` for `/v1/responses` +2. `dynamic_config.py` generates a responses config class (inherits from `OpenAIResponsesAPIConfig`) +3. `ProviderConfigManager.get_provider_responses_api_config()` returns the generated config +4. Request/response transformation is inherited from OpenAI — no custom code needed + ## Benefits - **Simple**: 2-5 lines of JSON vs 100+ lines of Python @@ -112,6 +139,10 @@ Use a Python config class if you need: - Provider-specific streaming logic - Advanced tool calling transformations +For providers that are *mostly* OpenAI-compatible but need small overrides (e.g. preset model handling), +you can inherit from `OpenAIResponsesAPIConfig` and override only what's needed — see +`litellm/llms/perplexity/responses/transformation.py` for a minimal example (~40 lines). + ## Implementation Details ### How It Works @@ -125,5 +156,6 @@ Use a Python config class if you need: The JSON system is integrated at: - `litellm/litellm_core_utils/get_llm_provider_logic.py` - Provider resolution -- `litellm/utils.py` - ProviderConfigManager +- `litellm/utils.py` - ProviderConfigManager (chat + responses) +- `litellm/responses/main.py` - Responses API routing - `litellm/constants.py` - openai_compatible_providers list diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index a2ce6b9a531..8be749f34a3 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -166,3 +166,63 @@ def create_config_class(provider: SimpleProviderConfig): return provider.slug return JSONProviderConfig + + +_responses_config_cache: dict = {} + + +def create_responses_config_class(provider: SimpleProviderConfig): + """Generate a Responses API config class dynamically from JSON configuration. + + Parallel to create_config_class() but for /v1/responses endpoints. + Classes are cached per provider slug to avoid regeneration on every request. + """ + if provider.slug in _responses_config_cache: + return _responses_config_cache[provider.slug] + + from litellm.llms.openai_like.responses.transformation import ( + OpenAILikeResponsesConfig, + ) + from litellm.types.router import GenericLiteLLMParams + + class JSONProviderResponsesConfig(OpenAILikeResponsesConfig): + @property + def custom_llm_provider(self): # type: ignore[override] + return provider.slug + + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams], + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or get_secret_str(provider.api_key_env) + ) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + if not api_base: + if provider.api_base_env: + api_base = get_secret_str(provider.api_base_env) + if not api_base: + api_base = provider.base_url + + if api_base is None: + raise ValueError( + f"api_base is required for provider {provider.slug}" + ) + + api_base = api_base.rstrip("/") + return f"{api_base}/responses" + + _responses_config_cache[provider.slug] = JSONProviderResponsesConfig + return JSONProviderResponsesConfig diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index f516d39662e..8b55fe4b618 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -21,6 +21,7 @@ class SimpleProviderConfig: self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) self.special_handling = data.get("special_handling", {}) + self.supported_endpoints = data.get("supported_endpoints", []) class JSONProviderRegistry: @@ -64,6 +65,14 @@ class JSONProviderRegistry: """Check if a provider is defined via JSON""" return slug in cls._providers + @classmethod + def supports_responses_api(cls, slug: str) -> bool: + """Check if a JSON provider supports the Responses API""" + provider = cls._providers.get(slug) + if provider is None: + return False + return "/v1/responses" in provider.supported_endpoints + @classmethod def list_providers(cls) -> list: """List all registered provider slugs""" diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 1b1b1c2f8cc..cae63c74a9a 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -80,6 +80,11 @@ "base_url": "https://api.gmi-serving.com/v1", "api_key_env": "GMI_API_KEY" }, + "perplexity": { + "base_url": "https://api.perplexity.ai", + "api_key_env": "PERPLEXITYAI_API_KEY", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + }, "sarvam": { "base_url": "https://api.sarvam.ai/v1", "api_key_env": "SARVAM_API_KEY", diff --git a/litellm/llms/openai_like/responses/__init__.py b/litellm/llms/openai_like/responses/__init__.py new file mode 100644 index 00000000000..e5421ec73d6 --- /dev/null +++ b/litellm/llms/openai_like/responses/__init__.py @@ -0,0 +1,5 @@ +from litellm.llms.openai_like.responses.transformation import ( + OpenAILikeResponsesConfig, +) + +__all__ = ["OpenAILikeResponsesConfig"] diff --git a/litellm/llms/openai_like/responses/transformation.py b/litellm/llms/openai_like/responses/transformation.py new file mode 100644 index 00000000000..ff496901363 --- /dev/null +++ b/litellm/llms/openai_like/responses/transformation.py @@ -0,0 +1,51 @@ +""" +OpenAI-like Responses API transformation. + +Base class for JSON-declared providers that support the /v1/responses endpoint. +Inherits everything from OpenAIResponsesAPIConfig; subclasses only override +provider-specific resolution (slug, API key env var, base URL). +""" + +from typing import Optional, Union + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class OpenAILikeResponsesConfig(OpenAIResponsesAPIConfig): + """ + Responses API config for OpenAI-compatible providers declared via JSON. + + Concrete per-provider classes are generated dynamically in dynamic_config.py. + This base provides the three overridable hooks that the dynamic generator + fills in: custom_llm_provider, validate_environment, get_complete_url. + """ + + @property + def custom_llm_provider(self) -> Union[str, LlmProviders]: # type: ignore[override] + return "openai_like" + + def validate_environment( + self, + headers: dict, + model: str, + litellm_params: Optional[GenericLiteLLMParams], + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = litellm_params.api_key or get_secret_str("OPENAI_LIKE_API_KEY") + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + api_base = api_base or get_secret_str("OPENAI_LIKE_API_BASE") + if not api_base: + raise ValueError("api_base is required for openai_like provider") + api_base = api_base.rstrip("/") + return f"{api_base}/responses" diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 178e76ea970..4beba088e7c 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -1,53 +1,26 @@ """ -Transformation logic for Perplexity Agentic Research API (Responses API) +Perplexity Responses API — OpenAI-compatible. -This module handles the translation between OpenAI's Responses API format -and Perplexity's Responses API format, which supports: -- Third-party model access (OpenAI, Anthropic, Google, xAI, etc.) -- Presets for optimized configurations -- Web search and URL fetching tools -- Reasoning effort control -- Instructions parameter for system-level guidance +The only provider quirks: +- cost returned as dict → handled by ResponseAPIUsage.parse_cost validator +- preset models (preset/pro-search) → handled by transform_responses_api_request + +Ref: https://docs.perplexity.ai/api-reference/responses-post """ -from typing import Any, Dict, List, Optional, Union +from typing import Dict, Optional, Union -import httpx - -from litellm._logging import verbose_logger -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ( - ResponseAPIUsage, - ResponseInputParam, - ResponsesAPIOptionalRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamingResponse, -) +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): - """ - Configuration for Perplexity Agentic Research API (Responses API) - - - Reference: https://docs.perplexity.ai/agentic-research/quickstart - """ - - @property - def custom_llm_provider(self) -> LlmProviders: - return LlmProviders.PERPLEXITY def get_supported_openai_params(self, model: str) -> list: - """ - Perplexity Responses API supports a different set of parameters - - Ref: https://docs.perplexity.ai/api-reference/responses-post - """ + """Ref: https://docs.perplexity.ai/api-reference/responses-post""" return [ "max_output_tokens", "stream", @@ -55,124 +28,29 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): "top_p", "tools", "reasoning", - "preset", "instructions", - "models", # Model fallback support ] + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.PERPLEXITY + def validate_environment( self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: - """Validate environment and set up headers""" - # Get API key from environment + litellm_params = litellm_params or GenericLiteLLMParams() api_key = ( - get_secret_str("PERPLEXITYAI_API_KEY") + litellm_params.api_key + or get_secret_str("PERPLEXITYAI_API_KEY") or get_secret_str("PERPLEXITY_API_KEY") ) - if api_key: headers["Authorization"] = f"Bearer {api_key}" - - headers["Content-Type"] = "application/json" - return headers - def get_complete_url( - self, - api_base: Optional[str], - litellm_params: dict, - ) -> str: - """Get the complete URL for the Perplexity Responses API""" - if api_base is None: - api_base = get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai" - - # Ensure api_base doesn't end with a slash - api_base = api_base.rstrip("/") - - # Add the responses endpoint - return f"{api_base}/v1/responses" - - def map_openai_params( - self, - response_api_optional_params: ResponsesAPIOptionalRequestParams, - model: str, - drop_params: bool, - ) -> Dict: - """ - Map OpenAI Responses API parameters to Perplexity format - - Key differences: - - Supports 'preset' parameter for predefined configurations - - Supports 'instructions' parameter for system-level guidance - - Tools are specified differently (web_search, fetch_url) - """ - mapped_params: Dict[str, Any] = {} - - # Map standard parameters - if response_api_optional_params.get("max_output_tokens"): - mapped_params["max_output_tokens"] = response_api_optional_params["max_output_tokens"] - - if response_api_optional_params.get("temperature"): - mapped_params["temperature"] = response_api_optional_params["temperature"] - - if response_api_optional_params.get("top_p"): - mapped_params["top_p"] = response_api_optional_params["top_p"] - - if response_api_optional_params.get("stream"): - mapped_params["stream"] = response_api_optional_params["stream"] - - if response_api_optional_params.get("stream_options"): - mapped_params["stream_options"] = response_api_optional_params["stream_options"] - - # Map Perplexity-specific parameters (using .get() with Any dict access) - preset = response_api_optional_params.get("preset") # type: ignore - if preset: - mapped_params["preset"] = preset - - instructions = response_api_optional_params.get("instructions") # type: ignore - if instructions: - mapped_params["instructions"] = instructions - - if response_api_optional_params.get("reasoning"): - mapped_params["reasoning"] = response_api_optional_params["reasoning"] - - tools = response_api_optional_params.get("tools") - if tools: - # Convert tools to list of dicts for transformation - tools_list = [dict(tool) if hasattr(tool, '__dict__') else tool for tool in tools] # type: ignore - mapped_params["tools"] = self._transform_tools(tools_list) # type: ignore - - return mapped_params - - def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Transform tools to Perplexity format - - Perplexity supports: - - web_search: Performs web searches - - fetch_url: Fetches content from URLs - """ - perplexity_tools = [] - - for tool in tools: - if isinstance(tool, dict): - tool_type = tool.get("type") - - # Direct Perplexity tool format - if tool_type in ["web_search", "fetch_url"]: - perplexity_tools.append(tool) - - # OpenAI function format - try to map to Perplexity tools - elif tool_type == "function": - function = tool.get("function", {}) - function_name = function.get("name", "") - - if function_name == "web_search" or "search" in function_name.lower(): - perplexity_tools.append({"type": "web_search"}) - elif function_name == "fetch_url" or "fetch" in function_name.lower(): - perplexity_tools.append({"type": "fetch_url"}) - - return perplexity_tools + def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str: + api_base = api_base or get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai" + return f"{api_base.rstrip('/')}/v1/responses" def transform_responses_api_request( self, @@ -182,228 +60,19 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> Dict: - """ - Transform request to Perplexity Responses API format - """ - # Check if the model is a preset (format: preset/preset-name) + """Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}.""" if model.startswith("preset/"): - preset_name = model.replace("preset/", "") - data = { - "preset": preset_name, - "input": self._format_input(input), + input = self._validate_input_param(input) + data: Dict = { + "preset": model[len("preset/"):], + "input": input, } - # Check if preset is explicitly provided in params - elif response_api_optional_request_params.get("preset"): - data = { - "preset": response_api_optional_request_params.pop("preset"), - "input": self._format_input(input), - } - else: - # Full request format for third-party models - data = { - "model": model, - "input": self._format_input(input), - } - - # Add all optional parameters - for key, value in response_api_optional_request_params.items(): - data[key] = value - - return data - - def _format_input(self, input: Union[str, ResponseInputParam]) -> Union[str, List[Dict[str, Any]]]: - """ - Format input for Perplexity Responses API - - The API accepts either: - - A simple string for single-turn queries - - An array of message objects for multi-turn conversations - """ - if isinstance(input, str): - return input - - # Handle ResponseInputParam format - if isinstance(input, list): - formatted_messages = [] - for item in input: - if isinstance(item, dict): - formatted_message = { - "type": "message", - "role": item.get("role"), - "content": item.get("content", ""), - } - formatted_messages.append(formatted_message) - return formatted_messages - - return str(input) - - def transform_response_api_response( - self, - model: str, - raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, - ) -> ResponsesAPIResponse: - """ - Transform Perplexity Responses API response to OpenAI Responses API format - """ - try: - raw_response_json = raw_response.json() - except Exception as e: - raise BaseLLMException( - status_code=raw_response.status_code, - message=f"Failed to parse response: {str(e)}", - ) - - # Check for error status - status = raw_response_json.get("status") - if status == "failed": - error = raw_response_json.get("error", {}) - error_message = error.get("message", "Unknown error") - raise BaseLLMException( - status_code=raw_response.status_code, - message=error_message, - ) - - # Transform usage to handle Perplexity's cost structure - usage_data = raw_response_json.get("usage", {}) - transformed_usage_dict = self._transform_usage(usage_data) - - # Convert usage dict to ResponseAPIUsage object - usage_obj = ResponseAPIUsage(**transformed_usage_dict) if transformed_usage_dict else None - - # Map Perplexity response to OpenAI Responses API format - response = ResponsesAPIResponse( - id=raw_response_json.get("id", ""), - object="response", - created_at=raw_response_json.get("created_at", 0), - status=raw_response_json.get("status", "completed"), - model=raw_response_json.get("model", model), - output=raw_response_json.get("output", []), - usage=usage_obj, + data.update(response_api_optional_request_params) + return data + return super().transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, ) - - return response - - def _transform_usage(self, usage_data: Dict[str, Any]) -> Dict[str, Any]: - """ - Transform Perplexity usage data to OpenAI format - - Perplexity returns: - { - "input_tokens": 100, - "output_tokens": 200, - "total_tokens": 300, - "cost": { - "currency": "USD", - "input_cost": 0.0001, - "output_cost": 0.0002, - "total_cost": 0.0003 - } - } - - OpenAI expects: - { - "input_tokens": 100, - "output_tokens": 200, - "total_tokens": 300, - "cost": 0.0003 - } - """ - transformed = { - "input_tokens": usage_data.get("input_tokens", 0), - "output_tokens": usage_data.get("output_tokens", 0), - "total_tokens": usage_data.get("total_tokens", 0), - } - - # Transform cost from Perplexity format (dict) to OpenAI format (float) - cost_obj = usage_data.get("cost") - if isinstance(cost_obj, dict) and "total_cost" in cost_obj: - transformed["cost"] = cost_obj["total_cost"] - verbose_logger.debug( - "Transformed Perplexity cost object to float: %s -> %s", - cost_obj, - cost_obj["total_cost"] - ) - elif cost_obj is not None: - # If cost is already a float/number, use it as-is - transformed["cost"] = cost_obj - - # Add input_tokens_details if present - if "input_tokens_details" in usage_data: - transformed["input_tokens_details"] = usage_data["input_tokens_details"] - - # Add output_tokens_details if present - if "output_tokens_details" in usage_data: - transformed["output_tokens_details"] = usage_data["output_tokens_details"] - - return transformed - - def transform_streaming_response( - self, - model: str, - parsed_chunk: dict, - logging_obj: LiteLLMLoggingObj, - ) -> ResponsesAPIStreamingResponse: - """ - Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse - """ - # Get the event type from the chunk - verbose_logger.debug("Raw Perplexity Chunk=%s", parsed_chunk) - event_type = str(parsed_chunk.get("type")) - event_pydantic_model = PerplexityResponsesConfig.get_event_model_class( - event_type=event_type - ) - - # Transform Perplexity-specific fields to OpenAI format - parsed_chunk = self._transform_perplexity_chunk(parsed_chunk) - - # Defensive: Handle error.code being null (similar to OpenAI implementation) - try: - error_obj = parsed_chunk.get("error") - if isinstance(error_obj, dict) and error_obj.get("code") is None: - # Preserve other fields, but ensure `code` is a non-null string - parsed_chunk = dict(parsed_chunk) - parsed_chunk["error"] = dict(error_obj) - parsed_chunk["error"]["code"] = "unknown_error" - except Exception: - # If anything unexpected happens here, fall back to attempting - # instantiation and let higher-level handlers manage errors. - verbose_logger.debug("Failed to coalesce error.code in parsed_chunk") - - return event_pydantic_model(**parsed_chunk) - - def _transform_perplexity_chunk(self, chunk: dict) -> dict: - """ - Transform Perplexity-specific fields in a streaming chunk to OpenAI format. - - This handles: - - Converting Perplexity's cost object to a simple float - """ - # Make a copy to avoid modifying the original - chunk = dict(chunk) - - # Transform usage.cost from Perplexity format to OpenAI format - # Perplexity: {"currency": "USD", "input_cost": 0.0001, "output_cost": 0.0002, "total_cost": 0.0003} - # OpenAI: 0.0003 (just the total_cost as a float) - try: - response_obj = chunk.get("response") - if isinstance(response_obj, dict): - usage_obj = response_obj.get("usage") - if isinstance(usage_obj, dict): - cost_obj = usage_obj.get("cost") - if isinstance(cost_obj, dict) and "total_cost" in cost_obj: - # Replace the cost object with just the total_cost value - chunk = dict(chunk) - chunk["response"] = dict(response_obj) - chunk["response"]["usage"] = dict(usage_obj) - chunk["response"]["usage"]["cost"] = cost_obj["total_cost"] - verbose_logger.debug( - "Transformed Perplexity cost object to float: %s -> %s", - cost_obj, - cost_obj["total_cost"] - ) - except Exception as e: - # If transformation fails, log and continue with original chunk - verbose_logger.debug("Failed to transform Perplexity cost object: %s", e) - - return chunk diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e943789a1cd..102798cc097 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -662,7 +662,7 @@ def responses( BaseResponsesAPIConfig ] = ProviderConfigManager.get_provider_responses_api_config( model=model, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) local_vars.update(kwargs) @@ -862,7 +862,7 @@ def delete_responses( BaseResponsesAPIConfig ] = ProviderConfigManager.get_provider_responses_api_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if responses_api_provider_config is None: @@ -1042,7 +1042,7 @@ def get_responses( BaseResponsesAPIConfig ] = ProviderConfigManager.get_provider_responses_api_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if responses_api_provider_config is None: @@ -1199,7 +1199,7 @@ def list_input_items( BaseResponsesAPIConfig ] = ProviderConfigManager.get_provider_responses_api_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if responses_api_provider_config is None: @@ -1357,7 +1357,7 @@ def cancel_responses( BaseResponsesAPIConfig ] = ProviderConfigManager.get_provider_responses_api_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if responses_api_provider_config is None: @@ -1544,7 +1544,7 @@ def compact_responses( BaseResponsesAPIConfig ] = ProviderConfigManager.get_provider_responses_api_config( model=model, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if responses_api_provider_config is None: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4ab81f8fd57..832e16be459 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1176,6 +1176,14 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): cost: Optional[float] = None """The cost of the request.""" + @field_validator("cost", mode="before") + @classmethod + def parse_cost(cls, v: Any) -> Optional[float]: + """Accept cost as a dict (e.g. Perplexity's {total_cost: 0.01}) and extract the float.""" + if isinstance(v, dict): + return v.get("total_cost") + return v + model_config = {"extra": "allow"} diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5f8798c7712..02c84beeb6e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1537,7 +1537,13 @@ class Usage(SafeAttributeModel, CompletionUsage): del self.server_tool_use if cost is not None: - self.cost = cost + # Some providers (e.g. Perplexity) return cost as a dict with total_cost + if isinstance(cost, dict): + cost = cost.get("total_cost") + if cost is not None: + self.cost = cost + else: + del self.cost else: del self.cost diff --git a/litellm/utils.py b/litellm/utils.py index 2a29c08904c..707c597c2b6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8226,9 +8226,52 @@ class ProviderConfigManager: @staticmethod def get_provider_responses_api_config( - provider: LlmProviders, + provider: Union[LlmProviders, str], model: Optional[str] = None, ) -> Optional[BaseResponsesAPIConfig]: + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + # Resolve provider string for JSON lookup + provider_str = provider.value if isinstance(provider, LlmProviders) else str(provider) + + # Try to convert to enum for Python class lookup first. + # Python classes take priority over JSON (they have custom overrides). + provider_enum: Optional[LlmProviders] = None + if isinstance(provider, LlmProviders): + provider_enum = provider + else: + try: + provider_enum = LlmProviders(provider) + except ValueError: + pass + + # Check Python classes first (custom overrides take priority) + result = ProviderConfigManager._get_python_responses_api_config( + provider_enum, model + ) + if result is not None: + return result + + # Fall back to JSON providers (generic OpenAI-compatible) + if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str): + provider_config = JSONProviderRegistry.get(provider_str) + if provider_config is not None: + return create_responses_config_class(provider_config)() + + return None + + @staticmethod + def _get_python_responses_api_config( + provider: Optional[LlmProviders], + model: Optional[str] = None, + ) -> Optional[BaseResponsesAPIConfig]: + """Check for Python-class-based responses API configs (custom overrides).""" + if provider is None: + return None + if litellm.LlmProviders.OPENAI == provider: return litellm.OpenAIResponsesAPIConfig() elif litellm.LlmProviders.AZURE == provider: diff --git a/tests/test_litellm/llms/openai_like/responses/__init__.py b/tests/test_litellm/llms/openai_like/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py b/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py new file mode 100644 index 00000000000..205b2343770 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py @@ -0,0 +1,337 @@ +""" +Tests for OpenAI-like Responses API support in the JSON provider system. +""" + +import os +import sys +from unittest.mock import patch + +import pytest + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + + +class TestSimpleProviderConfigSupportedEndpoints: + """Test the supported_endpoints field on SimpleProviderConfig.""" + + def test_default_supported_endpoints(self): + """supported_endpoints defaults to [] (chat always enabled, nothing else)""" + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + + config = SimpleProviderConfig("test", {"base_url": "https://example.com", "api_key_env": "TEST_KEY"}) + assert config.supported_endpoints == [] + + def test_custom_supported_endpoints(self): + """supported_endpoints can be set explicitly""" + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + + config = SimpleProviderConfig( + "test", + { + "base_url": "https://example.com", + "api_key_env": "TEST_KEY", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + }, + ) + assert "/v1/responses" in config.supported_endpoints + assert "/v1/chat/completions" in config.supported_endpoints + + def test_responses_only_endpoint(self): + """A provider can support only responses""" + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + + config = SimpleProviderConfig( + "test", + { + "base_url": "https://example.com", + "api_key_env": "TEST_KEY", + "supported_endpoints": ["/v1/responses"], + }, + ) + assert config.supported_endpoints == ["/v1/responses"] + + +class TestJSONProviderRegistryResponsesAPI: + """Test supports_responses_api on JSONProviderRegistry.""" + + def test_existing_provider_no_responses(self): + """Existing providers without supported_endpoints don't support responses""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + # publicai has no supported_endpoints in JSON, defaults to [] + assert JSONProviderRegistry.supports_responses_api("publicai") is False + + def test_nonexistent_provider(self): + """Non-existent provider returns False""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.supports_responses_api("nonexistent_provider_xyz") is False + + def test_provider_with_responses_endpoint(self): + """A provider with /v1/responses in supported_endpoints returns True""" + from litellm.llms.openai_like.json_loader import ( + JSONProviderRegistry, + SimpleProviderConfig, + ) + + # Temporarily inject a test provider + test_config = SimpleProviderConfig( + "test_responses_provider", + { + "base_url": "https://test.example.com", + "api_key_env": "TEST_API_KEY", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + }, + ) + JSONProviderRegistry._providers["test_responses_provider"] = test_config + try: + assert JSONProviderRegistry.supports_responses_api("test_responses_provider") is True + finally: + del JSONProviderRegistry._providers["test_responses_provider"] + + +class TestCreateResponsesConfigClass: + """Test dynamic responses config class generation.""" + + def _make_test_provider(self): + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + + return SimpleProviderConfig( + "test_resp", + { + "base_url": "https://api.testresp.com/v1", + "api_key_env": "TEST_RESP_API_KEY", + "api_base_env": "TEST_RESP_API_BASE", + "supported_endpoints": ["/v1/responses"], + }, + ) + + def test_generated_class_custom_llm_provider(self): + """Generated class returns the provider slug""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + assert config.custom_llm_provider == "test_resp" + + def test_generated_class_get_complete_url(self): + """Generated class builds correct responses URL""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + url = config.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://api.testresp.com/v1/responses" + + def test_generated_class_get_complete_url_with_override(self): + """api_base override takes precedence""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + url = config.get_complete_url(api_base="https://custom.api.com/v1", litellm_params={}) + assert url == "https://custom.api.com/v1/responses" + + def test_generated_class_get_complete_url_strips_trailing_slash(self): + """Trailing slashes are stripped from base URL""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + url = config.get_complete_url(api_base="https://custom.api.com/v1/", litellm_params={}) + assert url == "https://custom.api.com/v1/responses" + + def test_generated_class_validate_environment(self): + """validate_environment sets Authorization header from env""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + with patch( + "litellm.llms.openai_like.dynamic_config.get_secret_str", + return_value="sk-test-key-123", + ): + headers = config.validate_environment(headers={}, model="test-model", litellm_params=None) + assert headers["Authorization"] == "Bearer sk-test-key-123" + + def test_generated_class_validate_environment_litellm_params_override(self): + """api_key from litellm_params takes precedence over env""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + from litellm.types.router import GenericLiteLLMParams + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + litellm_params = GenericLiteLLMParams(api_key="sk-override-key") + headers = config.validate_environment( + headers={}, model="test-model", litellm_params=litellm_params + ) + assert headers["Authorization"] == "Bearer sk-override-key" + + def test_generated_class_inherits_openai_responses_methods(self): + """Generated class inherits OpenAI Responses API transformation methods""" + from litellm.llms.openai.responses.transformation import ( + OpenAIResponsesAPIConfig, + ) + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + # Should have inherited methods from OpenAIResponsesAPIConfig + assert hasattr(config, "get_supported_openai_params") + assert hasattr(config, "map_openai_params") + assert hasattr(config, "transform_responses_api_request") + assert hasattr(config, "transform_response_api_response") + assert hasattr(config, "transform_streaming_response") + + # Verify inheritance chain + assert isinstance(config, OpenAIResponsesAPIConfig) + + def test_generated_class_get_complete_url_uses_api_base_env(self): + """get_complete_url falls back to api_base_env when api_base is None""" + from litellm.llms.openai_like.dynamic_config import ( + create_responses_config_class, + ) + + provider = self._make_test_provider() + config_cls = create_responses_config_class(provider) + config = config_cls() + + with patch( + "litellm.llms.openai_like.dynamic_config.get_secret_str", + return_value="https://env-override.example.com/v1", + ): + url = config.get_complete_url(api_base=None, litellm_params={}) + assert url == "https://env-override.example.com/v1/responses" + + +class TestProviderConfigManagerResponsesAPI: + """Test that ProviderConfigManager integrates JSON responses providers.""" + + def test_json_provider_with_responses_returns_config(self): + """A JSON provider with /v1/responses returns a responses config""" + from litellm.llms.openai_like.json_loader import ( + JSONProviderRegistry, + SimpleProviderConfig, + ) + from litellm.utils import ProviderConfigManager + + test_config = SimpleProviderConfig( + "test_pcm_resp", + { + "base_url": "https://api.testpcm.com/v1", + "api_key_env": "TEST_PCM_KEY", + "supported_endpoints": ["/v1/responses"], + }, + ) + JSONProviderRegistry._providers["test_pcm_resp"] = test_config + try: + config = ProviderConfigManager.get_provider_responses_api_config( + provider="test_pcm_resp", + model="some-model", + ) + assert config is not None + assert config.custom_llm_provider == "test_pcm_resp" + finally: + del JSONProviderRegistry._providers["test_pcm_resp"] + + def test_json_provider_without_responses_returns_none(self): + """A JSON provider without /v1/responses returns None""" + from litellm.utils import ProviderConfigManager + + # publicai only supports chat completions + config = ProviderConfigManager.get_provider_responses_api_config( + provider="publicai", + model="some-model", + ) + assert config is None + + def test_unknown_provider_returns_none(self): + """A completely unknown provider returns None""" + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_responses_api_config( + provider="totally_unknown_provider_xyz", + model="some-model", + ) + assert config is None + + def test_standard_providers_still_work(self): + """Existing enum-based providers still resolve correctly""" + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_responses_api_config( + provider=LlmProviders.OPENAI, + model="gpt-4o", + ) + assert config is not None + + def test_standard_provider_as_string_still_works(self): + """Passing 'openai' as a string also works""" + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_responses_api_config( + provider="openai", + model="gpt-4o", + ) + assert config is not None + + def test_python_class_takes_priority_over_json(self): + """If a provider has both a Python class and JSON config, Python wins""" + from litellm.llms.openai_like.json_loader import ( + JSONProviderRegistry, + SimpleProviderConfig, + ) + from litellm.llms.perplexity.responses.transformation import ( + PerplexityResponsesConfig, + ) + from litellm.utils import ProviderConfigManager + + # Inject perplexity into JSON registry with responses support + test_config = SimpleProviderConfig( + "perplexity", + { + "base_url": "https://api.perplexity.ai", + "api_key_env": "PERPLEXITY_API_KEY", + "supported_endpoints": ["/v1/responses"], + }, + ) + JSONProviderRegistry._providers["perplexity"] = test_config + try: + config = ProviderConfigManager.get_provider_responses_api_config( + provider="perplexity", + model="some-model", + ) + # Should be the Python class, not the JSON-generated one + assert isinstance(config, PerplexityResponsesConfig) + finally: + del JSONProviderRegistry._providers["perplexity"]