diff --git a/.circleci/config.yml b/.circleci/config.yml index ecae22f872d..71580874453 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -168,7 +168,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -267,7 +267,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -511,7 +511,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -678,6 +678,48 @@ jobs: paths: - llm_translation_coverage.xml - llm_translation_coverage + llm_responses_api_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml llm_responses_api_coverage.xml + mv .coverage llm_responses_api_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - llm_responses_api_coverage.xml + - llm_responses_api_coverage litellm_mapped_tests: docker: - image: cimg/python:3.11 @@ -1234,7 +1276,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0 " + pip install "openai==1.66.1" - run: name: Install Grype command: | @@ -1309,7 +1351,7 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 120m # Store test results @@ -1370,7 +1412,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0 " + pip install "openai==1.66.1" # Run pytest and generate JUnit XML report - run: name: Build Docker image @@ -1492,7 +1534,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0 " + pip install "openai==1.66.1" - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -1921,7 +1963,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "google-cloud-aiplatform==1.43.0" pip install aiohttp - pip install "openai==1.54.0 " + pip install "openai==1.66.1" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip pip install "pydantic==2.7.1" @@ -2068,7 +2110,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage + coverage combine llm_translation_coverage llm_responses_api_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -2197,7 +2239,7 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install "openai==1.54.0 " + pip install "openai==1.66.1" python -m pip install --upgrade pip pip install "pydantic==2.7.1" pip install "pytest==7.3.1" @@ -2429,6 +2471,12 @@ workflows: only: - main - /litellm_.*/ + - llm_responses_api_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - litellm_mapped_tests: filters: branches: @@ -2468,6 +2516,7 @@ workflows: - upload-coverage: requires: - llm_translation_testing + - llm_responses_api_testing - litellm_mapped_tests - batches_testing - litellm_utils_testing @@ -2526,6 +2575,7 @@ workflows: - load_testing - test_bad_database_url - llm_translation_testing + - llm_responses_api_testing - litellm_mapped_tests - batches_testing - litellm_utils_testing diff --git a/litellm/__init__.py b/litellm/__init__.py index fd026ffb9d2..3ed4783951c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -922,6 +922,7 @@ from .llms.groq.chat.transformation import GroqChatConfig from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig from .llms.azure_ai.chat.transformation import AzureAIStudioConfig from .llms.mistral.mistral_chat_transformation import MistralConfig +from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility OpenAIOSeriesConfig, @@ -1011,6 +1012,7 @@ from .batches.main import * from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * +from .responses.main import * from .realtime_api.main import _arealtime from .fine_tuning.main import * from .files.main import * diff --git a/litellm/constants.py b/litellm/constants.py index 0288c45e40b..b4551a78f5f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -18,6 +18,7 @@ SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = 1000 # Minimum number of requests REPEATED_STREAMING_CHUNK_LIMIT = 100 # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. #### Networking settings #### request_timeout: float = 6000 # time in seconds +STREAM_SSE_DONE_STRING: str = "[DONE]" LITELLM_CHAT_PROVIDERS = [ "openai", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b83fe093055..58600ea14fd 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -44,7 +44,12 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_ai_image_cost_calculator, ) -from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.llms.openai import ( + HttpxBinaryResponseContent, + ResponseAPIUsage, + ResponsesAPIResponse, +) from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( CallTypesLiteral, @@ -464,6 +469,13 @@ def _get_usage_object( return usage_obj +def _is_known_usage_objects(usage_obj): + """Returns True if the usage obj is a known Usage type""" + return isinstance(usage_obj, litellm.Usage) or isinstance( + usage_obj, ResponseAPIUsage + ) + + def _infer_call_type( call_type: Optional[CallTypesLiteral], completion_response: Any ) -> Optional[CallTypesLiteral]: @@ -585,8 +597,8 @@ def completion_cost( # noqa: PLR0915 ) else: usage_obj = getattr(completion_response, "usage", {}) - if isinstance(usage_obj, BaseModel) and not isinstance( - usage_obj, litellm.Usage + if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects( + usage_obj=usage_obj ): setattr( completion_response, @@ -599,6 +611,14 @@ def completion_cost( # noqa: PLR0915 _usage = usage_obj.model_dump() else: _usage = usage_obj + + if ResponseAPILoggingUtils._is_response_api_usage(_usage): + _usage = ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _usage + ).model_dump() + ) + # get input/output tokens from completion_response prompt_tokens = _usage.get("prompt_tokens", 0) completion_tokens = _usage.get("completion_tokens", 0) @@ -797,6 +817,7 @@ def response_cost_calculator( TextCompletionResponse, HttpxBinaryResponseContent, RerankResponse, + ResponsesAPIResponse, ], model: str, custom_llm_provider: Optional[str], diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a3d9a57a497..18af6399180 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -39,11 +39,14 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, ) +from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( AllMessageValues, Batch, FineTuningJob, HttpxBinaryResponseContent, + ResponseCompletedEvent, + ResponsesAPIResponse, ) from litellm.types.rerank import RerankResponse from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS @@ -851,6 +854,8 @@ class Logging(LiteLLMLoggingBaseClass): RerankResponse, Batch, FineTuningJob, + ResponsesAPIResponse, + ResponseCompletedEvent, ], cache_hit: Optional[bool] = None, ) -> Optional[float]: @@ -1000,7 +1005,7 @@ class Logging(LiteLLMLoggingBaseClass): standard_logging_object is None and result is not None and self.stream is not True - ): # handle streaming separately + ): if ( isinstance(result, ModelResponse) or isinstance(result, ModelResponseStream) @@ -1012,6 +1017,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(result, RerankResponse) or isinstance(result, FineTuningJob) or isinstance(result, LiteLLMBatch) + or isinstance(result, ResponsesAPIResponse) ): ## HIDDEN PARAMS ## hidden_params = getattr(result, "_hidden_params", {}) @@ -1111,7 +1117,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] ] = None if "complete_streaming_response" in self.model_call_details: return # break out of this. @@ -1633,7 +1639,7 @@ class Logging(LiteLLMLoggingBaseClass): if "async_complete_streaming_response" in self.model_call_details: return # break out of this. complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] ] = self._get_assembled_streaming_response( result=result, start_time=start_time, @@ -2343,16 +2349,24 @@ class Logging(LiteLLMLoggingBaseClass): def _get_assembled_streaming_response( self, - result: Union[ModelResponse, TextCompletionResponse, ModelResponseStream, Any], + result: Union[ + ModelResponse, + TextCompletionResponse, + ModelResponseStream, + ResponseCompletedEvent, + Any, + ], start_time: datetime.datetime, end_time: datetime.datetime, is_async: bool, streaming_chunks: List[Any], - ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: + ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]]: if isinstance(result, ModelResponse): return result elif isinstance(result, TextCompletionResponse): return result + elif isinstance(result, ResponseCompletedEvent): + return result.response elif isinstance(result, ModelResponseStream): complete_streaming_response: Optional[ Union[ModelResponse, TextCompletionResponse] @@ -3111,6 +3125,12 @@ class StandardLoggingPayloadSetup: elif isinstance(usage, Usage): return usage elif isinstance(usage, dict): + if ResponseAPILoggingUtils._is_response_api_usage(usage): + return ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + ) return Usage(**usage) raise ValueError(f"usage is required, got={usage} of type {type(usage)}") diff --git a/litellm/llms/azure/assistants.py b/litellm/llms/azure/assistants.py index 2f67b5506f0..b446d887c40 100644 --- a/litellm/llms/azure/assistants.py +++ b/litellm/llms/azure/assistants.py @@ -1,4 +1,4 @@ -from typing import Coroutine, Iterable, Literal, Optional, Union +from typing import Any, Coroutine, Dict, Iterable, Literal, Optional, Union import httpx from openai import AsyncAzureOpenAI, AzureOpenAI @@ -618,7 +618,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -659,12 +659,12 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -684,12 +684,12 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AssistantStreamManager[AssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -711,7 +711,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -733,7 +733,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -756,7 +756,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py new file mode 100644 index 00000000000..c41d63842b1 --- /dev/null +++ b/litellm/llms/base_llm/responses/transformation.py @@ -0,0 +1,133 @@ +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, Optional, Union + +import httpx + +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + from ..chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException +else: + LiteLLMLoggingObj = Any + BaseLLMException = Any + + +class BaseResponsesAPIConfig(ABC): + def __init__(self): + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @abstractmethod + def get_supported_openai_params(self, model: str) -> list: + pass + + @abstractmethod + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + + pass + + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + return {} + + @abstractmethod + def get_complete_url( + self, + api_base: Optional[str], + model: str, + stream: Optional[bool] = None, + ) -> str: + """ + OPTIONAL + + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + + @abstractmethod + def transform_responses_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> ResponsesAPIRequestParams: + pass + + @abstractmethod + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + pass + + @abstractmethod + def transform_streaming_response( + self, + model: str, + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIStreamingResponse: + """ + Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse + """ + pass + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + from ..chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9d67fd1a853..6f8fd347c14 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,6 +1,6 @@ import io import json -from typing import TYPE_CHECKING, Any, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Union import httpx # type: ignore @@ -11,13 +11,21 @@ import litellm.types.utils from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, _get_httpx_client, get_async_httpx_client, ) +from litellm.responses.streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse from litellm.types.rerank import OptionalRerankParams, RerankResponse +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager @@ -952,8 +960,235 @@ class BaseLLMHTTPHandler: return returned_response return model_response + def response_api_handler( + self, + model: str, + input: Union[str, ResponseInputParam], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> Union[ + ResponsesAPIResponse, + BaseResponsesAPIStreamingIterator, + Coroutine[ + Any, Any, Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] + ], + ]: + """ + Handles responses API requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=response_api_optional_request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client if isinstance(client, AsyncHTTPHandler) else None, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + api_key=litellm_params.api_key, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + model=model, + ) + + data = responses_api_provider_config.transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + # Check if streaming is requested + stream = response_api_optional_request_params.get("stream", False) + + try: + if stream: + # For streaming, use stream=True in the request + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout + or response_api_optional_request_params.get("timeout"), + stream=True, + ) + + return SyncResponsesAPIStreamingIterator( + response=response, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + ) + else: + # For non-streaming requests + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout + or response_api_optional_request_params.get("timeout"), + ) + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_response_api_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_response_api_handler( + self, + model: str, + input: Union[str, ResponseInputParam], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: + """ + Async version of the responses API handler. + Uses async HTTP client to make requests. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + api_key=litellm_params.api_key, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + model=model, + ) + + data = responses_api_provider_config.transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + # Check if streaming is requested + stream = response_api_optional_request_params.get("stream", False) + + try: + if stream: + # For streaming, we need to use stream=True in the request + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout + or response_api_optional_request_params.get("timeout"), + stream=True, + ) + + # Return the streaming iterator + return ResponsesAPIStreamingIterator( + response=response, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + ) + else: + # For non-streaming, proceed as before + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout + or response_api_optional_request_params.get("timeout"), + ) + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_response_api_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + def _handle_error( - self, e: Exception, provider_config: Union[BaseConfig, BaseRerankConfig] + self, + e: Exception, + provider_config: Union[BaseConfig, BaseRerankConfig, BaseResponsesAPIConfig], ): status_code = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 7935c46293c..880a043d08a 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -2650,7 +2650,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -2689,12 +2689,12 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -2714,12 +2714,12 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AssistantStreamManager[AssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -2741,7 +2741,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -2763,7 +2763,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -2786,7 +2786,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py new file mode 100644 index 00000000000..ce4052dc197 --- /dev/null +++ b/litellm/llms/openai/responses/transformation.py @@ -0,0 +1,190 @@ +from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import * +from litellm.types.router import GenericLiteLLMParams + +from ..common_utils import OpenAIError + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): + def get_supported_openai_params(self, model: str) -> list: + """ + All OpenAI Responses API params are supported + """ + return [ + "input", + "model", + "include", + "instructions", + "max_output_tokens", + "metadata", + "parallel_tool_calls", + "previous_response_id", + "reasoning", + "store", + "stream", + "temperature", + "text", + "tool_choice", + "tools", + "top_p", + "truncation", + "user", + "extra_headers", + "extra_query", + "extra_body", + "timeout", + ] + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + """No mapping applied since inputs are in OpenAI spec already""" + return dict(response_api_optional_params) + + def transform_responses_api_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> ResponsesAPIRequestParams: + """No transform applied since inputs are in OpenAI spec already""" + return ResponsesAPIRequestParams( + model=model, input=input, **response_api_optional_request_params + ) + + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """No transform applied since outputs are in OpenAI spec already""" + try: + raw_response_json = raw_response.json() + except Exception: + raise OpenAIError( + message=raw_response.text, status_code=raw_response.status_code + ) + return ResponsesAPIResponse(**raw_response_json) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + headers.update( + { + "Authorization": f"Bearer {api_key}", + } + ) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + stream: Optional[bool] = None, + ) -> str: + """ + Get the endpoint for OpenAI responses API + """ + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + return f"{api_base}/responses" + + def transform_streaming_response( + self, + model: str, + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIStreamingResponse: + """ + Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse + """ + # Convert the dictionary to a properly typed ResponsesAPIStreamingResponse + verbose_logger.debug("Raw OpenAI Chunk=%s", parsed_chunk) + event_type = str(parsed_chunk.get("type")) + event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class( + event_type=event_type + ) + return event_pydantic_model(**parsed_chunk) + + @staticmethod + def get_event_model_class(event_type: str) -> Any: + """ + Returns the appropriate event model class based on the event type. + + Args: + event_type (str): The type of event from the response chunk + + Returns: + Any: The corresponding event model class + + Raises: + ValueError: If the event type is unknown + """ + event_models = { + ResponsesAPIStreamEvents.RESPONSE_CREATED: ResponseCreatedEvent, + ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS: ResponseInProgressEvent, + ResponsesAPIStreamEvents.RESPONSE_COMPLETED: ResponseCompletedEvent, + ResponsesAPIStreamEvents.RESPONSE_FAILED: ResponseFailedEvent, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE: ResponseIncompleteEvent, + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED: OutputItemAddedEvent, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: OutputItemDoneEvent, + ResponsesAPIStreamEvents.CONTENT_PART_ADDED: ContentPartAddedEvent, + ResponsesAPIStreamEvents.CONTENT_PART_DONE: ContentPartDoneEvent, + ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA: OutputTextDeltaEvent, + ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED: OutputTextAnnotationAddedEvent, + ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: OutputTextDoneEvent, + ResponsesAPIStreamEvents.REFUSAL_DELTA: RefusalDeltaEvent, + ResponsesAPIStreamEvents.REFUSAL_DONE: RefusalDoneEvent, + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA: FunctionCallArgumentsDeltaEvent, + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE: FunctionCallArgumentsDoneEvent, + ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS: FileSearchCallInProgressEvent, + ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING: FileSearchCallSearchingEvent, + ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED: FileSearchCallCompletedEvent, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS: WebSearchCallInProgressEvent, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING: WebSearchCallSearchingEvent, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED: WebSearchCallCompletedEvent, + ResponsesAPIStreamEvents.ERROR: ErrorEvent, + } + + model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type)) + if not model_class: + raise ValueError(f"Unknown event type: {event_type}") + + return model_class diff --git a/litellm/responses/main.py b/litellm/responses/main.py new file mode 100644 index 00000000000..ce70292e962 --- /dev/null +++ b/litellm/responses/main.py @@ -0,0 +1,217 @@ +import asyncio +import contextvars +from functools import partial +from typing import Any, Dict, Iterable, List, Literal, Optional, Union + +import httpx + +import litellm +from litellm.constants import request_timeout +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ( + Reasoning, + ResponseIncludable, + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponseTextConfigParam, + ToolChoice, + ToolParam, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.utils import ProviderConfigManager, client + +from .streaming_iterator import BaseResponsesAPIStreamingIterator + +####### ENVIRONMENT VARIABLES ################### +# Initialize any necessary instances or variables here +base_llm_http_handler = BaseLLMHTTPHandler() +################################################# + + +@client +async def aresponses( + input: Union[str, ResponseInputParam], + model: str, + include: Optional[List[ResponseIncludable]] = None, + instructions: Optional[str] = None, + max_output_tokens: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + parallel_tool_calls: Optional[bool] = None, + previous_response_id: Optional[str] = None, + reasoning: Optional[Reasoning] = None, + store: Optional[bool] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + text: Optional[ResponseTextConfigParam] = None, + tool_choice: Optional[ToolChoice] = None, + tools: Optional[Iterable[ToolParam]] = None, + top_p: Optional[float] = None, + truncation: Optional[Literal["auto", "disabled"]] = None, + user: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + **kwargs, +) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: + """ + Async: Handles responses API requests by reusing the synchronous function + """ + try: + loop = asyncio.get_event_loop() + kwargs["aresponses"] = True + + func = partial( + responses, + input=input, + model=model, + include=include, + instructions=instructions, + max_output_tokens=max_output_tokens, + metadata=metadata, + parallel_tool_calls=parallel_tool_calls, + previous_response_id=previous_response_id, + reasoning=reasoning, + store=store, + stream=stream, + temperature=temperature, + text=text, + tool_choice=tool_choice, + tools=tools, + top_p=top_p, + truncation=truncation, + user=user, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + return response + except Exception as e: + raise e + + +@client +def responses( + input: Union[str, ResponseInputParam], + model: str, + include: Optional[List[ResponseIncludable]] = None, + instructions: Optional[str] = None, + max_output_tokens: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + parallel_tool_calls: Optional[bool] = None, + previous_response_id: Optional[str] = None, + reasoning: Optional[Reasoning] = None, + store: Optional[bool] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + text: Optional[ResponseTextConfigParam] = None, + tool_choice: Optional[ToolChoice] = None, + tools: Optional[Iterable[ToolParam]] = None, + top_p: Optional[float] = None, + truncation: Optional[Literal["auto", "disabled"]] = None, + user: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + **kwargs, +): + """ + Synchronous version of the Responses API. + Uses the synchronous HTTP handler to make requests. + """ + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("aresponses", False) is True + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + model, custom_llm_provider, dynamic_api_key, dynamic_api_base = ( + litellm.get_llm_provider( + model=model, + custom_llm_provider=kwargs.get("custom_llm_provider", None), + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + ) + + # get provider config + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if responses_api_provider_config is None: + raise litellm.BadRequestError( + model=model, + llm_provider=custom_llm_provider, + message=f"Responses API not available for custom_llm_provider={custom_llm_provider}, model: {model}", + ) + + # Get all parameters using locals() and combine with kwargs + local_vars = locals() + local_vars.update(kwargs) + # Get ResponsesAPIOptionalRequestParams with only valid parameters + response_api_optional_params: ResponsesAPIOptionalRequestParams = ( + ResponsesAPIRequestUtils.get_requested_response_api_optional_param(local_vars) + ) + + # Get optional parameters for the responses API + responses_api_request_params: Dict = ( + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_params=response_api_optional_params, + ) + ) + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model=model, + user=user, + optional_params=dict(responses_api_request_params), + litellm_params={ + "litellm_call_id": litellm_call_id, + **responses_api_request_params, + }, + custom_llm_provider=custom_llm_provider, + ) + + # Call the handler with _is_async flag instead of directly calling the async handler + response = base_llm_http_handler.response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=responses_api_request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + return response diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py new file mode 100644 index 00000000000..c016e71e7e2 --- /dev/null +++ b/litellm/responses/streaming_iterator.py @@ -0,0 +1,209 @@ +import asyncio +import json +from datetime import datetime +from typing import Optional + +import httpx + +from litellm.constants import STREAM_SSE_DONE_STRING +from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.thread_pool_executor import executor +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponsesAPIStreamEvents, + ResponsesAPIStreamingResponse, +) +from litellm.utils import CustomStreamWrapper + + +class BaseResponsesAPIStreamingIterator: + """ + Base class for streaming iterators that process responses from the Responses API. + + This class contains shared logic for both synchronous and asynchronous iterators. + """ + + def __init__( + self, + response: httpx.Response, + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + logging_obj: LiteLLMLoggingObj, + ): + self.response = response + self.model = model + self.logging_obj = logging_obj + self.finished = False + self.responses_api_provider_config = responses_api_provider_config + self.completed_response: Optional[ResponsesAPIStreamingResponse] = None + self.start_time = datetime.now() + + def _process_chunk(self, chunk): + """Process a single chunk of data from the stream""" + if not chunk: + return None + + # Handle SSE format (data: {...}) + chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) + if chunk is None: + return None + + # Handle "[DONE]" marker + if chunk == STREAM_SSE_DONE_STRING: + self.finished = True + return None + + try: + # Parse the JSON chunk + parsed_chunk = json.loads(chunk) + + # Format as ResponsesAPIStreamingResponse + if isinstance(parsed_chunk, dict): + openai_responses_api_chunk = ( + self.responses_api_provider_config.transform_streaming_response( + model=self.model, + parsed_chunk=parsed_chunk, + logging_obj=self.logging_obj, + ) + ) + # Store the completed response + if ( + openai_responses_api_chunk + and openai_responses_api_chunk.type + == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ): + self.completed_response = openai_responses_api_chunk + self._handle_logging_completed_response() + + return openai_responses_api_chunk + + return None + except json.JSONDecodeError: + # If we can't parse the chunk, continue + return None + + def _handle_logging_completed_response(self): + """Base implementation - should be overridden by subclasses""" + pass + + +class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): + """ + Async iterator for processing streaming responses from the Responses API. + """ + + def __init__( + self, + response: httpx.Response, + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + logging_obj: LiteLLMLoggingObj, + ): + super().__init__(response, model, responses_api_provider_config, logging_obj) + self.stream_iterator = response.aiter_lines() + + def __aiter__(self): + return self + + async def __anext__(self) -> ResponsesAPIStreamingResponse: + try: + while True: + # Get the next chunk from the stream + try: + chunk = await self.stream_iterator.__anext__() + except StopAsyncIteration: + self.finished = True + raise StopAsyncIteration + + result = self._process_chunk(chunk) + + if self.finished: + raise StopAsyncIteration + elif result is not None: + return result + # If result is None, continue the loop to get the next chunk + + except httpx.HTTPError as e: + # Handle HTTP errors + self.finished = True + raise e + + def _handle_logging_completed_response(self): + """Handle logging for completed responses in async context""" + asyncio.create_task( + self.logging_obj.async_success_handler( + result=self.completed_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) + ) + + executor.submit( + self.logging_obj.success_handler, + result=self.completed_response, + cache_hit=None, + start_time=self.start_time, + end_time=datetime.now(), + ) + + +class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): + """ + Synchronous iterator for processing streaming responses from the Responses API. + """ + + def __init__( + self, + response: httpx.Response, + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + logging_obj: LiteLLMLoggingObj, + ): + super().__init__(response, model, responses_api_provider_config, logging_obj) + self.stream_iterator = response.iter_lines() + + def __iter__(self): + return self + + def __next__(self): + try: + while True: + # Get the next chunk from the stream + try: + chunk = next(self.stream_iterator) + except StopIteration: + self.finished = True + raise StopIteration + + result = self._process_chunk(chunk) + + if self.finished: + raise StopIteration + elif result is not None: + return result + # If result is None, continue the loop to get the next chunk + + except httpx.HTTPError as e: + # Handle HTTP errors + self.finished = True + raise e + + def _handle_logging_completed_response(self): + """Handle logging for completed responses in sync context""" + run_async_function( + async_function=self.logging_obj.async_success_handler, + result=self.completed_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) + + executor.submit( + self.logging_obj.success_handler, + result=self.completed_response, + cache_hit=None, + start_time=self.start_time, + end_time=datetime.now(), + ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py new file mode 100644 index 00000000000..49d850ec6aa --- /dev/null +++ b/litellm/responses/utils.py @@ -0,0 +1,97 @@ +from typing import Any, Dict, cast, get_type_hints + +import litellm +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponsesAPIOptionalRequestParams, +) +from litellm.types.utils import Usage + + +class ResponsesAPIRequestUtils: + """Helper utils for constructing ResponseAPI requests""" + + @staticmethod + def get_optional_params_responses_api( + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + ) -> Dict: + """ + Get optional parameters for the responses API. + + Args: + params: Dictionary of all parameters + model: The model name + responses_api_provider_config: The provider configuration for responses API + + Returns: + A dictionary of supported parameters for the responses API + """ + # Remove None values and internal parameters + + # Get supported parameters for the model + supported_params = responses_api_provider_config.get_supported_openai_params( + model + ) + + # Check for unsupported parameters + unsupported_params = [ + param + for param in response_api_optional_params + if param not in supported_params + ] + + if unsupported_params: + raise litellm.UnsupportedParamsError( + model=model, + message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", + ) + + # Map parameters to provider-specific format + mapped_params = responses_api_provider_config.map_openai_params( + response_api_optional_params=response_api_optional_params, + model=model, + drop_params=litellm.drop_params, + ) + + return mapped_params + + @staticmethod + def get_requested_response_api_optional_param( + params: Dict[str, Any] + ) -> ResponsesAPIOptionalRequestParams: + """ + Filter parameters to only include those defined in ResponsesAPIOptionalRequestParams. + + Args: + params: Dictionary of parameters to filter + + Returns: + ResponsesAPIOptionalRequestParams instance with only the valid parameters + """ + valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() + filtered_params = {k: v for k, v in params.items() if k in valid_keys} + return cast(ResponsesAPIOptionalRequestParams, filtered_params) + + +class ResponseAPILoggingUtils: + @staticmethod + def _is_response_api_usage(usage: dict) -> bool: + """returns True if usage is from OpenAI Response API""" + if "input_tokens" in usage and "output_tokens" in usage: + return True + return False + + @staticmethod + def _transform_response_api_usage_to_chat_usage(usage: dict) -> Usage: + """Tranforms the ResponseAPIUsage object to a Usage object""" + response_api_usage: ResponseAPIUsage = ResponseAPIUsage(**usage) + prompt_tokens: int = response_api_usage.input_tokens or 0 + completion_tokens: int = response_api_usage.output_tokens or 0 + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index ee017744354..5b811830e70 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,6 +1,8 @@ +from enum import Enum from os import PathLike from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union +import httpx from openai._legacy_response import ( HttpxBinaryResponseContent as _HttpxBinaryResponseContent, ) @@ -31,8 +33,24 @@ from openai.types.chat.chat_completion_prediction_content_param import ( ) from openai.types.embedding import Embedding as OpenAIEmbedding from openai.types.fine_tuning.fine_tuning_job import FineTuningJob -from pydantic import BaseModel, Field -from typing_extensions import Dict, Required, TypedDict, override +from openai.types.responses.response import ( + IncompleteDetails, + Response, + ResponseOutputItem, + ResponseTextConfig, + Tool, + ToolChoice, +) +from openai.types.responses.response_create_params import ( + Reasoning, + ResponseIncludable, + ResponseInputParam, + ResponseTextConfigParam, + ToolChoice, + ToolParam, +) +from pydantic import BaseModel, Discriminator, Field, PrivateAttr +from typing_extensions import Annotated, Dict, Required, TypedDict, override FileContent = Union[IO[bytes], bytes, PathLike] @@ -684,3 +702,323 @@ OpenAIAudioTranscriptionOptionalParams = Literal[ OpenAIImageVariationOptionalParams = Literal["n", "size", "response_format", "user"] + + +class ResponsesAPIOptionalRequestParams(TypedDict, total=False): + """TypedDict for Optional parameters supported by the responses API.""" + + include: Optional[List[ResponseIncludable]] + instructions: Optional[str] + max_output_tokens: Optional[int] + metadata: Optional[Dict[str, Any]] + parallel_tool_calls: Optional[bool] + previous_response_id: Optional[str] + reasoning: Optional[Reasoning] + store: Optional[bool] + stream: Optional[bool] + temperature: Optional[float] + text: Optional[ResponseTextConfigParam] + tool_choice: Optional[ToolChoice] + tools: Optional[Iterable[ToolParam]] + top_p: Optional[float] + truncation: Optional[Literal["auto", "disabled"]] + user: Optional[str] + + +class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): + """TypedDict for request parameters supported by the responses API.""" + + input: Union[str, ResponseInputParam] + model: str + + +class BaseLiteLLMOpenAIResponseObject(BaseModel): + def __getitem__(self, key): + return self.__dict__[key] + + def get(self, key, default=None): + return self.__dict__.get(key, default) + + def __contains__(self, key): + return key in self.__dict__ + + +class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): + reasoning_tokens: int + + model_config = {"extra": "allow"} + + +class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): + input_tokens: int + """The number of input tokens.""" + + output_tokens: int + """The number of output tokens.""" + + output_tokens_details: Optional[OutputTokensDetails] + """A detailed breakdown of the output tokens.""" + + total_tokens: int + """The total number of tokens used.""" + + model_config = {"extra": "allow"} + + +class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): + id: str + created_at: float + error: Optional[dict] + incomplete_details: Optional[IncompleteDetails] + instructions: Optional[str] + metadata: Optional[Dict] + model: Optional[str] + object: Optional[str] + output: List[ResponseOutputItem] + parallel_tool_calls: bool + temperature: Optional[float] + tool_choice: ToolChoice + tools: List[Tool] + top_p: Optional[float] + max_output_tokens: Optional[int] + previous_response_id: Optional[str] + reasoning: Optional[Reasoning] + status: Optional[str] + text: Optional[ResponseTextConfig] + truncation: Optional[Literal["auto", "disabled"]] + usage: Optional[ResponseAPIUsage] + user: Optional[str] + # Define private attributes using PrivateAttr + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class ResponsesAPIStreamEvents(str, Enum): + """ + Enum representing all supported OpenAI stream event types for the Responses API. + + Inherits from str to allow direct string comparison and usage as dictionary keys. + """ + + # Response lifecycle events + RESPONSE_CREATED = "response.created" + RESPONSE_IN_PROGRESS = "response.in_progress" + RESPONSE_COMPLETED = "response.completed" + RESPONSE_FAILED = "response.failed" + RESPONSE_INCOMPLETE = "response.incomplete" + + # Output item events + OUTPUT_ITEM_ADDED = "response.output_item.added" + OUTPUT_ITEM_DONE = "response.output_item.done" + + # Content part events + CONTENT_PART_ADDED = "response.content_part.added" + CONTENT_PART_DONE = "response.content_part.done" + + # Output text events + OUTPUT_TEXT_DELTA = "response.output_text.delta" + OUTPUT_TEXT_ANNOTATION_ADDED = "response.output_text.annotation.added" + OUTPUT_TEXT_DONE = "response.output_text.done" + + # Refusal events + REFUSAL_DELTA = "response.refusal.delta" + REFUSAL_DONE = "response.refusal.done" + + # Function call events + FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" + FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" + + # File search events + FILE_SEARCH_CALL_IN_PROGRESS = "response.file_search_call.in_progress" + FILE_SEARCH_CALL_SEARCHING = "response.file_search_call.searching" + FILE_SEARCH_CALL_COMPLETED = "response.file_search_call.completed" + + # Web search events + WEB_SEARCH_CALL_IN_PROGRESS = "response.web_search_call.in_progress" + WEB_SEARCH_CALL_SEARCHING = "response.web_search_call.searching" + WEB_SEARCH_CALL_COMPLETED = "response.web_search_call.completed" + + # Error event + ERROR = "error" + + +class ResponseCreatedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED] + response: ResponsesAPIResponse + + +class ResponseInProgressEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS] + response: ResponsesAPIResponse + + +class ResponseCompletedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + response: ResponsesAPIResponse + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class ResponseFailedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_FAILED] + response: ResponsesAPIResponse + + +class ResponseIncompleteEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] + response: ResponsesAPIResponse + + +class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] + output_index: int + item: dict + + +class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] + output_index: int + item: dict + + +class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] + item_id: str + output_index: int + content_index: int + part: dict + + +class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] + item_id: str + output_index: int + content_index: int + part: dict + + +class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] + item_id: str + output_index: int + content_index: int + delta: str + + +class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED] + item_id: str + output_index: int + content_index: int + annotation_index: int + annotation: dict + + +class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] + item_id: str + output_index: int + content_index: int + text: str + + +class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.REFUSAL_DELTA] + item_id: str + output_index: int + content_index: int + delta: str + + +class RefusalDoneEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.REFUSAL_DONE] + item_id: str + output_index: int + content_index: int + refusal: str + + +class FunctionCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] + item_id: str + output_index: int + delta: str + + +class FunctionCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE] + item_id: str + output_index: int + arguments: str + + +class FileSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS] + output_index: int + item_id: str + + +class FileSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING] + output_index: int + item_id: str + + +class FileSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED] + output_index: int + item_id: str + + +class WebSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS] + output_index: int + item_id: str + + +class WebSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING] + output_index: int + item_id: str + + +class WebSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED] + output_index: int + item_id: str + + +class ErrorEvent(BaseLiteLLMOpenAIResponseObject): + type: Literal[ResponsesAPIStreamEvents.ERROR] + code: Optional[str] + message: str + param: Optional[str] + + +# Union type for all possible streaming responses +ResponsesAPIStreamingResponse = Annotated[ + Union[ + ResponseCreatedEvent, + ResponseInProgressEvent, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + OutputItemAddedEvent, + OutputItemDoneEvent, + ContentPartAddedEvent, + ContentPartDoneEvent, + OutputTextDeltaEvent, + OutputTextAnnotationAddedEvent, + OutputTextDoneEvent, + RefusalDeltaEvent, + RefusalDoneEvent, + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDoneEvent, + FileSearchCallInProgressEvent, + FileSearchCallSearchingEvent, + FileSearchCallCompletedEvent, + WebSearchCallInProgressEvent, + WebSearchCallSearchingEvent, + WebSearchCallCompletedEvent, + ErrorEvent, + ], + Discriminator("type"), +] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4af88100faf..04b76f3a2e4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -191,6 +191,8 @@ class CallTypes(Enum): retrieve_batch = "retrieve_batch" pass_through = "pass_through_endpoint" anthropic_messages = "anthropic_messages" + responses = "responses" + aresponses = "aresponses" CallTypesLiteral = Literal[ diff --git a/litellm/utils.py b/litellm/utils.py index 2f1cac743c1..18fc70b1103 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -209,6 +209,7 @@ from litellm.llms.base_llm.image_variations.transformation import ( BaseImageVariationConfig, ) from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from ._logging import _is_debugging_on, verbose_logger from .caching.caching import ( @@ -715,6 +716,11 @@ def function_setup( # noqa: PLR0915 call_type == CallTypes.aspeech.value or call_type == CallTypes.speech.value ): messages = kwargs.get("input", "speech") + elif ( + call_type == CallTypes.aresponses.value + or call_type == CallTypes.responses.value + ): + messages = args[0] if len(args) > 0 else kwargs["input"] else: messages = "default-message-value" stream = True if "stream" in kwargs and kwargs["stream"] is True else False @@ -5104,7 +5110,7 @@ def prompt_token_calculator(model, messages): from anthropic import AI_PROMPT, HUMAN_PROMPT, Anthropic anthropic_obj = Anthropic() - num_tokens = anthropic_obj.count_tokens(text) + num_tokens = anthropic_obj.count_tokens(text) # type: ignore else: num_tokens = len(encoding.encode(text)) return num_tokens @@ -6276,6 +6282,15 @@ class ProviderConfigManager: return litellm.DeepgramAudioTranscriptionConfig() return None + @staticmethod + def get_provider_responses_api_config( + model: str, + provider: LlmProviders, + ) -> Optional[BaseResponsesAPIConfig]: + if litellm.LlmProviders.OPENAI == provider: + return litellm.OpenAIResponsesAPIConfig() + return None + @staticmethod def get_provider_text_completion_config( model: str, diff --git a/requirements.txt b/requirements.txt index 3d695d17662..dcdddff1179 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # anyio==4.4.0 # openai + http req. httpx==0.27.0 # Pin Httpx dependency -openai==1.61.0 # openai req. +openai==1.66.1 # openai req. fastapi==0.115.5 # server dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep diff --git a/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py new file mode 100644 index 00000000000..b4a6cd974ef --- /dev/null +++ b/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -0,0 +1,239 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) + + +class TestOpenAIResponsesAPIConfig: + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + self.model = "gpt-4o" + self.logging_obj = MagicMock() + + def test_map_openai_params(self): + """Test that parameters are correctly mapped""" + test_params = {"input": "Hello world", "temperature": 0.7, "stream": True} + + result = self.config.map_openai_params( + response_api_optional_params=test_params, + model=self.model, + drop_params=False, + ) + + # The function should return the params unchanged + assert result == test_params + + def validate_responses_api_request_params(self, params, expected_fields): + """ + Validate that the params dict has the expected structure of ResponsesAPIRequestParams + + Args: + params: The dict to validate + expected_fields: Dict of field names and their expected values + """ + # Check that it's a dict + assert isinstance(params, dict), "Result should be a dict" + + # Check expected fields have correct values + for field, value in expected_fields.items(): + assert field in params, f"Missing expected field: {field}" + assert ( + params[field] == value + ), f"Field {field} has value {params[field]}, expected {value}" + + def test_transform_responses_api_request(self): + """Test request transformation""" + input_text = "What is the capital of France?" + optional_params = {"temperature": 0.7, "stream": True} + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_text, + response_api_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + # Validate the result has the expected structure and values + expected_fields = { + "model": self.model, + "input": input_text, + "temperature": 0.7, + "stream": True, + } + + self.validate_responses_api_request_params(result, expected_fields) + + def test_transform_streaming_response(self): + """Test streaming response transformation""" + # Test with a text delta event + chunk = { + "type": "response.output_text.delta", + "item_id": "item_123", + "output_index": 0, + "content_index": 0, + "delta": "Hello", + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputTextDeltaEvent) + assert result.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + assert result.delta == "Hello" + assert result.item_id == "item_123" + + # Test with a completed event - providing all required fields + completed_chunk = { + "type": "response.completed", + "response": { + "id": "resp_123", + "created_at": 1234567890, + "model": "gpt-4o", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": 0.7, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": None, + "truncation": "auto", + "usage": None, + "user": None, + }, + } + + # Mock the get_event_model_class to avoid validation issues in tests + with patch.object( + OpenAIResponsesAPIConfig, "get_event_model_class" + ) as mock_get_class: + mock_get_class.return_value = ResponseCompletedEvent + + result = self.config.transform_streaming_response( + model=self.model, + parsed_chunk=completed_chunk, + logging_obj=self.logging_obj, + ) + + assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + assert result.response.id == "resp_123" + + def test_validate_environment(self): + """Test that validate_environment correctly sets the Authorization header""" + # Test with provided API key + headers = {} + api_key = "test_api_key" + + result = self.config.validate_environment( + headers=headers, model=self.model, api_key=api_key + ) + + assert "Authorization" in result + assert result["Authorization"] == f"Bearer {api_key}" + + # Test with empty headers + headers = {} + + with patch("litellm.api_key", "litellm_api_key"): + result = self.config.validate_environment(headers=headers, model=self.model) + + assert "Authorization" in result + assert result["Authorization"] == "Bearer litellm_api_key" + + # Test with existing headers + headers = {"Content-Type": "application/json"} + + with patch("litellm.openai_key", "openai_key"): + with patch("litellm.api_key", None): + result = self.config.validate_environment( + headers=headers, model=self.model + ) + + assert "Authorization" in result + assert result["Authorization"] == "Bearer openai_key" + assert "Content-Type" in result + assert result["Content-Type"] == "application/json" + + # Test with environment variable + headers = {} + + with patch("litellm.api_key", None): + with patch("litellm.openai_key", None): + with patch( + "litellm.llms.openai.responses.transformation.get_secret_str", + return_value="env_api_key", + ): + result = self.config.validate_environment( + headers=headers, model=self.model + ) + + assert "Authorization" in result + assert result["Authorization"] == "Bearer env_api_key" + + def test_get_complete_url(self): + """Test that get_complete_url returns the correct URL""" + # Test with provided API base + api_base = "https://custom-openai.example.com/v1" + + result = self.config.get_complete_url(api_base=api_base, model=self.model) + + assert result == "https://custom-openai.example.com/v1/responses" + + # Test with litellm.api_base + with patch("litellm.api_base", "https://litellm-api-base.example.com/v1"): + result = self.config.get_complete_url(api_base=None, model=self.model) + + assert result == "https://litellm-api-base.example.com/v1/responses" + + # Test with environment variable + with patch("litellm.api_base", None): + with patch( + "litellm.llms.openai.responses.transformation.get_secret_str", + return_value="https://env-api-base.example.com/v1", + ): + result = self.config.get_complete_url(api_base=None, model=self.model) + + assert result == "https://env-api-base.example.com/v1/responses" + + # Test with default API base + with patch("litellm.api_base", None): + with patch( + "litellm.llms.openai.responses.transformation.get_secret_str", + return_value=None, + ): + result = self.config.get_complete_url(api_base=None, model=self.model) + + assert result == "https://api.openai.com/v1/responses" + + # Test with trailing slash in API base + api_base = "https://custom-openai.example.com/v1/" + + result = self.config.get_complete_url(api_base=api_base, model=self.model) + + assert result == "https://custom-openai.example.com/v1/responses" diff --git a/tests/litellm/responses/test_responses_utils.py b/tests/litellm/responses/test_responses_utils.py new file mode 100644 index 00000000000..3567f609e73 --- /dev/null +++ b/tests/litellm/responses/test_responses_utils.py @@ -0,0 +1,150 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.utils import Usage + + +class TestResponsesAPIRequestUtils: + def test_get_optional_params_responses_api(self): + """Test that optional parameters are correctly processed for responses API""" + # Setup + model = "gpt-4o" + config = OpenAIResponsesAPIConfig() + optional_params = ResponsesAPIOptionalRequestParams( + {"temperature": 0.7, "max_output_tokens": 100} + ) + + # Execute + result = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=config, + response_api_optional_params=optional_params, + ) + + # Assert + assert result == optional_params + assert "temperature" in result + assert result["temperature"] == 0.7 + assert "max_output_tokens" in result + assert result["max_output_tokens"] == 100 + + def test_get_optional_params_responses_api_unsupported_param(self): + """Test that unsupported parameters raise an error""" + # Setup + model = "gpt-4o" + config = OpenAIResponsesAPIConfig() + optional_params = ResponsesAPIOptionalRequestParams( + {"temperature": 0.7, "unsupported_param": "value"} + ) + + # Execute and Assert + with pytest.raises(litellm.UnsupportedParamsError) as excinfo: + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=config, + response_api_optional_params=optional_params, + ) + + assert "unsupported_param" in str(excinfo.value) + assert model in str(excinfo.value) + + def test_get_requested_response_api_optional_param(self): + """Test filtering parameters to only include those in ResponsesAPIOptionalRequestParams""" + # Setup + params = { + "temperature": 0.7, + "max_output_tokens": 100, + "invalid_param": "value", + "model": "gpt-4o", # This is not in ResponsesAPIOptionalRequestParams + } + + # Execute + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + params + ) + + # Assert + assert "temperature" in result + assert "max_output_tokens" in result + assert "invalid_param" not in result + assert "model" not in result + assert result["temperature"] == 0.7 + assert result["max_output_tokens"] == 100 + + +class TestResponseAPILoggingUtils: + def test_is_response_api_usage_true(self): + """Test identification of Response API usage format""" + # Setup + usage = {"input_tokens": 10, "output_tokens": 20} + + # Execute + result = ResponseAPILoggingUtils._is_response_api_usage(usage) + + # Assert + assert result is True + + def test_is_response_api_usage_false(self): + """Test identification of non-Response API usage format""" + # Setup + usage = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + + # Execute + result = ResponseAPILoggingUtils._is_response_api_usage(usage) + + # Assert + assert result is False + + def test_transform_response_api_usage_to_chat_usage(self): + """Test transformation from Response API usage to Chat usage format""" + # Setup + usage = { + "input_tokens": 10, + "output_tokens": 20, + "total_tokens": 30, + "output_tokens_details": {"reasoning_tokens": 5}, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert + assert isinstance(result, Usage) + assert result.prompt_tokens == 10 + assert result.completion_tokens == 20 + assert result.total_tokens == 30 + + def test_transform_response_api_usage_with_none_values(self): + """Test transformation handles None values properly""" + # Setup + usage = { + "input_tokens": 0, # Changed from None to 0 + "output_tokens": 20, + "total_tokens": 20, + "output_tokens_details": {"reasoning_tokens": 5}, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert + assert result.prompt_tokens == 0 + assert result.completion_tokens == 20 + assert result.total_tokens == 20 diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py new file mode 100644 index 00000000000..b3561d8a626 --- /dev/null +++ b/tests/llm_responses_api_testing/conftest.py @@ -0,0 +1,63 @@ +# conftest.py + +import importlib +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm + + +@pytest.fixture(scope="function", autouse=True) +def setup_and_teardown(): + """ + This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. + """ + curr_dir = os.getcwd() # Get the current working directory + sys.path.insert( + 0, os.path.abspath("../..") + ) # Adds the project directory to the system path + + import litellm + from litellm import Router + + importlib.reload(litellm) + + try: + if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): + import litellm.proxy.proxy_server + + importlib.reload(litellm.proxy.proxy_server) + except Exception as e: + print(f"Error reloading litellm.proxy.proxy_server: {e}") + + import asyncio + + loop = asyncio.get_event_loop_policy().new_event_loop() + asyncio.set_event_loop(loop) + print(litellm) + # from litellm import Router, completion, aembedding, acompletion, embedding + yield + + # Teardown code (executes after the yield point) + loop.close() # Close the loop created earlier + asyncio.set_event_loop(None) # Remove the reference to the loop + + +def pytest_collection_modifyitems(config, items): + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + custom_logger_tests = [ + item for item in items if "custom_logger" in item.parent.name + ] + other_tests = [item for item in items if "custom_logger" not in item.parent.name] + + # Sort tests based on their names + custom_logger_tests.sort(key=lambda x: x.name) + other_tests.sort(key=lambda x: x.name) + + # Reorder the items list + items[:] = custom_logger_tests + other_tests diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py new file mode 100644 index 00000000000..ca62c664d30 --- /dev/null +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -0,0 +1,505 @@ +import os +import sys +import pytest +import asyncio +from typing import Optional + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm.integrations.custom_logger import CustomLogger +import json +from litellm.types.utils import StandardLoggingPayload +from litellm.types.llms.openai import ( + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponseTextConfig, + ResponseAPIUsage, + IncompleteDetails, +) + + +def validate_responses_api_response(response, final_chunk: bool = False): + """ + Validate that a response from litellm.responses() or litellm.aresponses() + conforms to the expected ResponsesAPIResponse structure. + + Args: + response: The response object to validate + + Raises: + AssertionError: If the response doesn't match the expected structure + """ + # Validate response structure + print("response=", json.dumps(response, indent=4, default=str)) + assert isinstance( + response, ResponsesAPIResponse + ), "Response should be an instance of ResponsesAPIResponse" + + # Required fields + assert "id" in response and isinstance( + response["id"], str + ), "Response should have a string 'id' field" + assert "created_at" in response and isinstance( + response["created_at"], (int, float) + ), "Response should have a numeric 'created_at' field" + assert "output" in response and isinstance( + response["output"], list + ), "Response should have a list 'output' field" + assert "parallel_tool_calls" in response and isinstance( + response["parallel_tool_calls"], bool + ), "Response should have a boolean 'parallel_tool_calls' field" + + # Optional fields with their expected types + optional_fields = { + "error": (dict, type(None)), # error can be dict or None + "incomplete_details": (IncompleteDetails, type(None)), + "instructions": (str, type(None)), + "metadata": dict, + "model": str, + "object": str, + "temperature": (int, float), + "tool_choice": (dict, str), + "tools": list, + "top_p": (int, float), + "max_output_tokens": (int, type(None)), + "previous_response_id": (str, type(None)), + "reasoning": dict, + "status": str, + "text": ResponseTextConfig, + "truncation": str, + "usage": ResponseAPIUsage, + "user": (str, type(None)), + } + if final_chunk is False: + optional_fields["usage"] = type(None) + + for field, expected_type in optional_fields.items(): + if field in response: + assert isinstance( + response[field], expected_type + ), f"Field '{field}' should be of type {expected_type}, but got {type(response[field])}" + + # Check if output has at least one item + if final_chunk is True: + assert ( + len(response["output"]) > 0 + ), "Response 'output' field should have at least one item" + + return True # Return True if validation passes + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_basic_openai_responses_api(sync_mode): + litellm._turn_on_debug() + + if sync_mode: + response = litellm.responses( + model="gpt-4o", input="Basic ping", max_output_tokens=20 + ) + else: + response = await litellm.aresponses( + model="gpt-4o", input="Basic ping", max_output_tokens=20 + ) + + print("litellm response=", json.dumps(response, indent=4, default=str)) + + # Use the helper function to validate the response + validate_responses_api_response(response, final_chunk=True) + + +@pytest.mark.parametrize("sync_mode", [True]) +@pytest.mark.asyncio +async def test_basic_openai_responses_api_streaming(sync_mode): + litellm._turn_on_debug() + + if sync_mode: + response = litellm.responses( + model="gpt-4o", + input="Basic ping", + stream=True, + ) + for event in response: + print("litellm response=", json.dumps(event, indent=4, default=str)) + else: + response = await litellm.aresponses( + model="gpt-4o", + input="Basic ping", + stream=True, + ) + async for event in response: + print("litellm response=", json.dumps(event, indent=4, default=str)) + + +class TestCustomLogger(CustomLogger): + def __init__( + self, + ): + self.standard_logging_object: Optional[StandardLoggingPayload] = None + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + print("in async_log_success_event") + print("kwargs=", json.dumps(kwargs, indent=4, default=str)) + self.standard_logging_object = kwargs["standard_logging_object"] + pass + + +def validate_standard_logging_payload( + slp: StandardLoggingPayload, response: ResponsesAPIResponse, request_model: str +): + """ + Validate that a StandardLoggingPayload object matches the expected response + + Args: + slp (StandardLoggingPayload): The standard logging payload object to validate + response (dict): The litellm response to compare against + request_model (str): The model name that was requested + """ + # Validate payload exists + assert slp is not None, "Standard logging payload should not be None" + + # Validate token counts + print("response=", json.dumps(response, indent=4, default=str)) + assert ( + slp["prompt_tokens"] == response["usage"]["input_tokens"] + ), "Prompt tokens mismatch" + assert ( + slp["completion_tokens"] == response["usage"]["output_tokens"] + ), "Completion tokens mismatch" + assert ( + slp["total_tokens"] + == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] + ), "Total tokens mismatch" + + # Validate spend and response metadata + assert slp["response_cost"] > 0, "Response cost should be greater than 0" + assert slp["id"] == response["id"], "Response ID mismatch" + assert slp["model"] == request_model, "Model name mismatch" + + # Validate messages + assert slp["messages"] == [{"content": "hi", "role": "user"}], "Messages mismatch" + + # Validate complete response structure + validate_responses_match(slp["response"], response) + + +@pytest.mark.asyncio +async def test_basic_openai_responses_api_streaming_with_logging(): + litellm._turn_on_debug() + litellm.set_verbose = True + test_custom_logger = TestCustomLogger() + litellm.callbacks = [test_custom_logger] + request_model = "gpt-4o" + response = await litellm.aresponses( + model=request_model, + input="hi", + stream=True, + ) + final_response: Optional[ResponseCompletedEvent] = None + async for event in response: + if event.type == "response.completed": + final_response = event + print("litellm response=", json.dumps(event, indent=4, default=str)) + + print("sleeping for 2 seconds...") + await asyncio.sleep(2) + print( + "standard logging payload=", + json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), + ) + + assert final_response is not None + assert test_custom_logger.standard_logging_object is not None + + validate_standard_logging_payload( + slp=test_custom_logger.standard_logging_object, + response=final_response.response, + request_model=request_model, + ) + + +def validate_responses_match(slp_response, litellm_response): + """Validate that the standard logging payload OpenAI response matches the litellm response""" + # Validate core fields + assert slp_response["id"] == litellm_response["id"], "ID mismatch" + assert slp_response["model"] == litellm_response["model"], "Model mismatch" + assert ( + slp_response["created_at"] == litellm_response["created_at"] + ), "Created at mismatch" + + # Validate usage + assert ( + slp_response["usage"]["input_tokens"] + == litellm_response["usage"]["input_tokens"] + ), "Input tokens mismatch" + assert ( + slp_response["usage"]["output_tokens"] + == litellm_response["usage"]["output_tokens"] + ), "Output tokens mismatch" + assert ( + slp_response["usage"]["total_tokens"] + == litellm_response["usage"]["total_tokens"] + ), "Total tokens mismatch" + + # Validate output/messages + assert len(slp_response["output"]) == len( + litellm_response["output"] + ), "Output length mismatch" + for slp_msg, litellm_msg in zip(slp_response["output"], litellm_response["output"]): + assert slp_msg["role"] == litellm_msg.role, "Message role mismatch" + # Access the content's text field for the litellm response + litellm_content = litellm_msg.content[0].text if litellm_msg.content else "" + assert ( + slp_msg["content"][0]["text"] == litellm_content + ), f"Message content mismatch. Expected {litellm_content}, Got {slp_msg['content']}" + assert slp_msg["status"] == litellm_msg.status, "Message status mismatch" + + +@pytest.mark.asyncio +async def test_basic_openai_responses_api_non_streaming_with_logging(): + litellm._turn_on_debug() + litellm.set_verbose = True + test_custom_logger = TestCustomLogger() + litellm.callbacks = [test_custom_logger] + request_model = "gpt-4o" + response = await litellm.aresponses( + model=request_model, + input="hi", + ) + + print("litellm response=", json.dumps(response, indent=4, default=str)) + print("response hidden params=", response._hidden_params) + + print("sleeping for 2 seconds...") + await asyncio.sleep(2) + print( + "standard logging payload=", + json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), + ) + + assert response is not None + assert test_custom_logger.standard_logging_object is not None + + validate_standard_logging_payload( + test_custom_logger.standard_logging_object, response, request_model + ) + + +def validate_stream_event(event): + """ + Validate that a streaming event from litellm.responses() or litellm.aresponses() + with stream=True conforms to the expected structure based on its event type. + + Args: + event: The streaming event object to validate + + Raises: + AssertionError: If the event doesn't match the expected structure for its type + """ + # Common validation for all event types + assert hasattr(event, "type"), "Event should have a 'type' attribute" + + # Type-specific validation + if event.type == "response.created" or event.type == "response.in_progress": + assert hasattr( + event, "response" + ), f"{event.type} event should have a 'response' attribute" + validate_responses_api_response(event.response, final_chunk=False) + + elif event.type == "response.completed": + assert hasattr( + event, "response" + ), "response.completed event should have a 'response' attribute" + validate_responses_api_response(event.response, final_chunk=True) + # Usage is guaranteed only on the completed event + assert ( + "usage" in event.response + ), "response.completed event should have usage information" + print("Usage in event.response=", event.response["usage"]) + assert isinstance(event.response["usage"], ResponseAPIUsage) + elif event.type == "response.failed" or event.type == "response.incomplete": + assert hasattr( + event, "response" + ), f"{event.type} event should have a 'response' attribute" + + elif ( + event.type == "response.output_item.added" + or event.type == "response.output_item.done" + ): + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "item" + ), f"{event.type} event should have an 'item' attribute" + + elif ( + event.type == "response.content_part.added" + or event.type == "response.content_part.done" + ): + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "part" + ), f"{event.type} event should have a 'part' attribute" + + elif event.type == "response.output_text.delta": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "delta" + ), f"{event.type} event should have a 'delta' attribute" + + elif event.type == "response.output_text.annotation.added": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "annotation_index" + ), f"{event.type} event should have an 'annotation_index' attribute" + assert hasattr( + event, "annotation" + ), f"{event.type} event should have an 'annotation' attribute" + + elif event.type == "response.output_text.done": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "text" + ), f"{event.type} event should have a 'text' attribute" + + elif event.type == "response.refusal.delta": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "delta" + ), f"{event.type} event should have a 'delta' attribute" + + elif event.type == "response.refusal.done": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "refusal" + ), f"{event.type} event should have a 'refusal' attribute" + + elif event.type == "response.function_call_arguments.delta": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "delta" + ), f"{event.type} event should have a 'delta' attribute" + + elif event.type == "response.function_call_arguments.done": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "arguments" + ), f"{event.type} event should have an 'arguments' attribute" + + elif event.type in [ + "response.file_search_call.in_progress", + "response.file_search_call.searching", + "response.file_search_call.completed", + "response.web_search_call.in_progress", + "response.web_search_call.searching", + "response.web_search_call.completed", + ]: + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + + elif event.type == "error": + assert hasattr( + event, "message" + ), "Error event should have a 'message' attribute" + return True # Return True if validation passes + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_openai_responses_api_streaming_validation(sync_mode): + """Test that validates each streaming event from the responses API""" + litellm._turn_on_debug() + + event_types_seen = set() + + if sync_mode: + response = litellm.responses( + model="gpt-4o", + input="Tell me about artificial intelligence in 3 sentences.", + stream=True, + ) + for event in response: + print(f"Validating event type: {event.type}") + validate_stream_event(event) + event_types_seen.add(event.type) + else: + response = await litellm.aresponses( + model="gpt-4o", + input="Tell me about artificial intelligence in 3 sentences.", + stream=True, + ) + async for event in response: + print(f"Validating event type: {event.type}") + validate_stream_event(event) + event_types_seen.add(event.type) + + # At minimum, we should see these core event types + required_events = {"response.created", "response.completed"} + + missing_events = required_events - event_types_seen + assert not missing_events, f"Missing required event types: {missing_events}" + + print(f"Successfully validated all event types: {event_types_seen}")