From 8e30f1fb2f07606f9c23bbee5ba3b2cce878010a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 13:49:43 -0700 Subject: [PATCH 01/61] new test for llm_responses_api_coverage --- .circleci/config.yml | 54 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ecae22f872d..68f9ee14cda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -678,6 +678,48 @@ jobs: paths: - llm_translation_coverage.xml - llm_translation_coverage + llm_responses_api_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml llm_responses_api_coverage.xml + mv .coverage llm_responses_api_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - llm_responses_api_testing.xml + - llm_responses_api_testing litellm_mapped_tests: docker: - image: cimg/python:3.11 @@ -1309,7 +1351,7 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 120m # Store test results @@ -2068,7 +2110,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage + coverage combine llm_translation_coverage llm_responses_api_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -2429,6 +2471,12 @@ workflows: only: - main - /litellm_.*/ + - llm_responses_api_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - litellm_mapped_tests: filters: branches: @@ -2468,6 +2516,7 @@ workflows: - upload-coverage: requires: - llm_translation_testing + - llm_responses_api_testing - litellm_mapped_tests - batches_testing - litellm_utils_testing @@ -2526,6 +2575,7 @@ workflows: - load_testing - test_bad_database_url - llm_translation_testing + - llm_responses_api_testing - litellm_mapped_tests - batches_testing - litellm_utils_testing From 4b1b87eb674d97541a5fcb7132eb71c7757120b1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 14:28:47 -0700 Subject: [PATCH 02/61] openai reasoning initial types --- litellm/responses/main.py | 45 +++++++++++++++++++ litellm/types/llms/openai.py | 8 ++++ .../test_openai_responses_api.py | 12 +++++ 3 files changed, 65 insertions(+) create mode 100644 litellm/responses/main.py create mode 100644 tests/llm_responses_api_testing/test_openai_responses_api.py diff --git a/litellm/responses/main.py b/litellm/responses/main.py new file mode 100644 index 00000000000..2d66f8e616b --- /dev/null +++ b/litellm/responses/main.py @@ -0,0 +1,45 @@ +from typing import Any, Dict, Iterable, List, Literal, Optional, Union + +import httpx + +from litellm.types.llms.openai import ( + Reasoning, + ResponseIncludable, + ResponseInputParam, + ResponseTextConfigParam, + ToolChoice, + ToolParam, +) + + +async def aresponses(): + pass + + +def responses( + input: Union[str, ResponseInputParam], + model: str, + include: Optional[List[ResponseIncludable]] = None, + instructions: Optional[str] = None, + max_output_tokens: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + parallel_tool_calls: Optional[bool] = None, + previous_response_id: Optional[str] = None, + reasoning: Optional[Reasoning] = None, + store: Optional[bool] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + text: Optional[ResponseTextConfigParam] = None, + tool_choice: Optional[ToolChoice] = None, + tools: Optional[Iterable[ToolParam]] = None, + top_p: Optional[float] = None, + truncation: Optional[Literal["auto", "disabled"]] = None, + user: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, +): + pass diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index ee017744354..0923a3b7167 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -31,6 +31,14 @@ from openai.types.chat.chat_completion_prediction_content_param import ( ) from openai.types.embedding import Embedding as OpenAIEmbedding from openai.types.fine_tuning.fine_tuning_job import FineTuningJob +from openai.types.responses.response_create_params import ( + Reasoning, + ResponseIncludable, + ResponseInputParam, + ResponseTextConfigParam, + ToolChoice, + ToolParam, +) from pydantic import BaseModel, Field from typing_extensions import Dict, Required, TypedDict, override diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py new file mode 100644 index 00000000000..74b60d35370 --- /dev/null +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -0,0 +1,12 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../..")) + + +def test_basic_openai_responses_api(): + response = litellm.responses( + model="gpt-4o", input="Tell me a three sentence bedtime story about a unicorn." + ) + + validate_responses_api_response() From d6c82327e69f34827a7db8259bc40eeb83ca6bd4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 14:32:32 -0700 Subject: [PATCH 03/61] working import litellm.responses --- litellm/__init__.py | 1 + tests/llm_responses_api_testing/test_openai_responses_api.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index d66707f8b3a..9e30a1c5762 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1010,6 +1010,7 @@ from .batches.main import * from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * +from .responses.main import * from .realtime_api.main import _arealtime from .fine_tuning.main import * from .files.main import * diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 74b60d35370..67581cec008 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -2,6 +2,7 @@ import os import sys sys.path.insert(0, os.path.abspath("../..")) +import litellm def test_basic_openai_responses_api(): @@ -9,4 +10,4 @@ def test_basic_openai_responses_api(): model="gpt-4o", input="Tell me a three sentence bedtime story about a unicorn." ) - validate_responses_api_response() + # validate_responses_api_response() From 8c4331638efe4c5abee4043fad20c828961bc671 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 14:33:50 -0700 Subject: [PATCH 04/61] add aysnc aresponses --- litellm/responses/main.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 2d66f8e616b..96a99615c21 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -12,7 +12,32 @@ from litellm.types.llms.openai import ( ) -async def aresponses(): +async def aresponses( + input: Union[str, ResponseInputParam], + model: str, + include: Optional[List[ResponseIncludable]] = None, + instructions: Optional[str] = None, + max_output_tokens: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + parallel_tool_calls: Optional[bool] = None, + previous_response_id: Optional[str] = None, + reasoning: Optional[Reasoning] = None, + store: Optional[bool] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + text: Optional[ResponseTextConfigParam] = None, + tool_choice: Optional[ToolChoice] = None, + tools: Optional[Iterable[ToolParam]] = None, + top_p: Optional[float] = None, + truncation: Optional[Literal["auto", "disabled"]] = None, + user: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, +): pass From 6462466e020fc3a68fc80041f29812b887dc9e03 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 15:09:42 -0700 Subject: [PATCH 05/61] add OpenAIResponsesAPIConfig --- .../llms/base_llm/responses/transformation.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 litellm/llms/base_llm/responses/transformation.py diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py new file mode 100644 index 00000000000..d80deed29c0 --- /dev/null +++ b/litellm/llms/base_llm/responses/transformation.py @@ -0,0 +1,97 @@ +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Optional + +from litellm.types.utils import ModelInfo + +from ..chat.transformation import BaseLLMException + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class BaseResponsesAPIConfig(ABC): + def __init__(self): + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @abstractmethod + def get_supported_openai_params(self, model: str) -> list: + pass + + @abstractmethod + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + pass + + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + optional_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + pass + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + OPTIONAL + + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + + @abstractmethod + def transform_request( + self, + model: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + pass + + @abstractmethod + def transform_response( + self, + ): + pass From 368f1de2e19db7568262691080d394840d0197be Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 15:10:34 -0700 Subject: [PATCH 06/61] add OpenAIResponsesAPIConfig --- litellm/__init__.py | 1 + litellm/llms/openai/responses/transformation.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 litellm/llms/openai/responses/transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 9e30a1c5762..dfb890a0b85 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -921,6 +921,7 @@ from .llms.groq.chat.transformation import GroqChatConfig from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig from .llms.azure_ai.chat.transformation import AzureAIStudioConfig from .llms.mistral.mistral_chat_transformation import MistralConfig +from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility OpenAIOSeriesConfig, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py new file mode 100644 index 00000000000..401df0ca183 --- /dev/null +++ b/litellm/llms/openai/responses/transformation.py @@ -0,0 +1,5 @@ +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig + + +class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): + pass From 401a52e6943358c608494a715d156b71ca4cb773 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 15:24:42 -0700 Subject: [PATCH 07/61] working transform --- .../llms/base_llm/responses/transformation.py | 5 +- .../llms/openai/responses/transformation.py | 60 ++++++++++++++++++- litellm/types/llms/openai.py | 28 +++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index d80deed29c0..b332084f685 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -2,6 +2,7 @@ import types from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Optional +from litellm.types.llms.openai import ResponsesAPIRequestParams from litellm.types.utils import ModelInfo from ..chat.transformation import BaseLLMException @@ -44,11 +45,11 @@ class BaseResponsesAPIConfig(ABC): @abstractmethod def map_openai_params( self, - non_default_params: dict, optional_params: dict, model: str, drop_params: bool, - ) -> dict: + ) -> ResponsesAPIRequestParams: + pass @abstractmethod diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 401df0ca183..bff4e12dfb5 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,5 +1,63 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.types.llms.openai import ResponsesAPIRequestParams class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): - pass + def get_supported_openai_params(self, model: str) -> list: + """ + All OpenAI Responses API params are supported + """ + return [ + "input", + "model", + "include", + "instructions", + "max_output_tokens", + "metadata", + "parallel_tool_calls", + "previous_response_id", + "reasoning", + "store", + "stream", + "temperature", + "text", + "tool_choice", + "tools", + "top_p", + "truncation", + "user", + "extra_headers", + "extra_query", + "extra_body", + "timeout", + ] + + def map_openai_params( + self, + optional_params: dict, + model: str, + drop_params: bool, + ) -> ResponsesAPIRequestParams: + + return ResponsesAPIRequestParams( + include=optional_params.get("include"), + instructions=optional_params.get("instructions"), + max_output_tokens=optional_params.get("max_output_tokens"), + metadata=optional_params.get("metadata"), + parallel_tool_calls=optional_params.get("parallel_tool_calls"), + previous_response_id=optional_params.get("previous_response_id"), + reasoning=optional_params.get("reasoning"), + store=optional_params.get("store"), + stream=optional_params.get("stream"), + temperature=optional_params.get("temperature"), + text=optional_params.get("text"), + tool_choice=optional_params.get("tool_choice"), + tools=optional_params.get("tools"), + top_p=optional_params.get("top_p"), + truncation=optional_params.get("truncation"), + user=optional_params.get("user"), + extra_headers=optional_params.get("extra_headers"), + extra_query=optional_params.get("extra_query"), + extra_body=optional_params.get("extra_body"), + timeout=optional_params.get("timeout"), + ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0923a3b7167..9b6581773d6 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,6 +1,7 @@ from os import PathLike from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union +import httpx from openai._legacy_response import ( HttpxBinaryResponseContent as _HttpxBinaryResponseContent, ) @@ -692,3 +693,30 @@ OpenAIAudioTranscriptionOptionalParams = Literal[ OpenAIImageVariationOptionalParams = Literal["n", "size", "response_format", "user"] + + +class ResponsesAPIRequestParams(TypedDict, total=False): + """TypedDict for parameters supported by the responses API.""" + + input: Union[str, ResponseInputParam] + model: str + include: Optional[List[ResponseIncludable]] + instructions: Optional[str] + max_output_tokens: Optional[int] + metadata: Optional[Dict[str, Any]] + parallel_tool_calls: Optional[bool] + previous_response_id: Optional[str] + reasoning: Optional[Reasoning] + store: Optional[bool] + stream: Optional[bool] + temperature: Optional[float] + text: Optional[ResponseTextConfigParam] + tool_choice: Optional[ToolChoice] + tools: Optional[Iterable[ToolParam]] + top_p: Optional[float] + truncation: Optional[Literal["auto", "disabled"]] + user: Optional[str] + extra_headers: Optional[Dict[str, Any]] + extra_query: Optional[Dict[str, Any]] + extra_body: Optional[Dict[str, Any]] + timeout: Optional[Union[float, httpx.Timeout]] From 4d55212c6292d9f0052d208f51ea11de195063fa Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 15:57:53 -0700 Subject: [PATCH 08/61] add BaseResponsesAPIConfig --- .../llms/base_llm/responses/transformation.py | 39 +++++++------------ litellm/utils.py | 12 +++++- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index b332084f685..669558f0eb3 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -52,17 +52,6 @@ class BaseResponsesAPIConfig(ABC): pass - @abstractmethod - def validate_environment( - self, - headers: dict, - model: str, - optional_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - ) -> dict: - pass - def get_complete_url( self, api_base: Optional[str], @@ -81,18 +70,18 @@ class BaseResponsesAPIConfig(ABC): raise ValueError("api_base is required") return api_base - @abstractmethod - def transform_request( - self, - model: str, - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: - pass + # @abstractmethod + # def transform_request( + # self, + # model: str, + # optional_params: dict, + # litellm_params: dict, + # headers: dict, + # ) -> dict: + # pass - @abstractmethod - def transform_response( - self, - ): - pass + # @abstractmethod + # def transform_response( + # self, + # ): + # pass diff --git a/litellm/utils.py b/litellm/utils.py index ce5acbc694b..97c8171b362 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -209,6 +209,7 @@ from litellm.llms.base_llm.image_variations.transformation import ( BaseImageVariationConfig, ) from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from ._logging import _is_debugging_on, verbose_logger from .caching.caching import ( @@ -5103,7 +5104,7 @@ def prompt_token_calculator(model, messages): from anthropic import AI_PROMPT, HUMAN_PROMPT, Anthropic anthropic_obj = Anthropic() - num_tokens = anthropic_obj.count_tokens(text) + num_tokens = anthropic_obj.count_tokens(text) # type: ignore else: num_tokens = len(encoding.encode(text)) return num_tokens @@ -6275,6 +6276,15 @@ class ProviderConfigManager: return litellm.DeepgramAudioTranscriptionConfig() return None + @staticmethod + def get_provider_responses_api_config( + model: str, + provider: LlmProviders, + ) -> Optional[BaseResponsesAPIConfig]: + if litellm.LlmProviders.OPENAI == provider: + return litellm.OpenAIResponsesAPIConfig() + return None + @staticmethod def get_provider_text_completion_config( model: str, From 2c6774e3ee157a458793ac569dd84a25458ee524 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:00:49 -0700 Subject: [PATCH 09/61] get_optional_params_responses_api --- litellm/llms/custom_httpx/llm_http_handler.py | 11 ++++ litellm/responses/main.py | 62 +++++++++++++++++++ litellm/responses/utils.py | 46 ++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 litellm/responses/utils.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9d67fd1a853..b8b7af686d3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -11,12 +11,14 @@ import litellm.types.utils from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, _get_httpx_client, get_async_httpx_client, ) +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIRequestParams from litellm.types.rerank import OptionalRerankParams, RerankResponse from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager @@ -952,6 +954,15 @@ class BaseLLMHTTPHandler: return returned_response return model_response + async def async_response_api_handler( + self, + model: str, + input: Union[str, ResponseInputParam], + responses_api_provider_config: BaseResponsesAPIConfig, + responses_api_request_params: ResponsesAPIRequestParams, + ) -> Any: + pass + def _handle_error( self, e: Exception, provider_config: Union[BaseConfig, BaseRerankConfig] ): diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 96a99615c21..0920d25d170 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -2,6 +2,13 @@ from typing import Any, Dict, Iterable, List, Literal, Optional, Union import httpx +import litellm +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.responses.utils import ( + ResponsesAPIRequestParams, + get_optional_params_responses_api, +) from litellm.types.llms.openai import ( Reasoning, ResponseIncludable, @@ -10,6 +17,13 @@ from litellm.types.llms.openai import ( ToolChoice, ToolParam, ) +from litellm.types.router import GenericLiteLLMParams +from litellm.utils import ProviderConfigManager + +####### ENVIRONMENT VARIABLES ################### +# Initialize any necessary instances or variables here +base_llm_http_handler = BaseLLMHTTPHandler() +################################################# async def aresponses( @@ -37,7 +51,55 @@ async def aresponses( extra_query: Optional[Dict[str, Any]] = None, extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, + **kwargs, ): + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + model, custom_llm_provider, dynamic_api_key, dynamic_api_base = ( + litellm.get_llm_provider( + model=model, + custom_llm_provider=kwargs.get("custom_llm_provider", None), + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + ) + + # get provider config + responses_api_provider_config: Optional[BaseResponsesAPIConfig] = ( + ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if responses_api_provider_config is None: + raise litellm.BadRequestError( + model=model, + llm_provider=custom_llm_provider, + message=f"Responses API not available for custom_llm_provider={custom_llm_provider}, model: {model}", + ) + + # Get all parameters using locals() and combine with kwargs + all_params = {**locals(), **kwargs} + + # Get optional parameters for the responses API + responses_api_request_params: ResponsesAPIRequestParams = ( + get_optional_params_responses_api( + model=model, + responses_api_provider_config=responses_api_provider_config, + optional_params={**locals(), **kwargs}, + ) + ) + + response = await base_llm_http_handler.async_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + responses_api_request_params=responses_api_request_params, + ) + return response + pass diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py new file mode 100644 index 00000000000..51a77bc23c9 --- /dev/null +++ b/litellm/responses/utils.py @@ -0,0 +1,46 @@ +from typing import Any, Dict + +import litellm +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.types.llms.openai import ResponsesAPIRequestParams + + +def get_optional_params_responses_api( + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + optional_params: Dict[str, Any], +) -> ResponsesAPIRequestParams: + """ + Get optional parameters for the responses API. + + Args: + params: Dictionary of all parameters + model: The model name + responses_api_provider_config: The provider configuration for responses API + + Returns: + A dictionary of supported parameters for the responses API + """ + # Remove None values and internal parameters + filtered_params = {k: v for k, v in optional_params.items() if v is not None} + + # Get supported parameters for the model + supported_params = responses_api_provider_config.get_supported_openai_params(model) + + # Check for unsupported parameters + unsupported_params = [ + param for param in filtered_params if param not in supported_params + ] + + if unsupported_params: + raise litellm.UnsupportedParamsError( + model=model, + message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", + ) + + # Map parameters to provider-specific format + mapped_params = responses_api_provider_config.map_openai_params( + optional_params=filtered_params, model=model, drop_params=litellm.drop_params + ) + + return mapped_params From 980354b78bfcda1247989ce854f51a68d3a2a16f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:12:36 -0700 Subject: [PATCH 10/61] add validate_environment, get_complete_url --- .../llms/openai/responses/transformation.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index bff4e12dfb5..7c52679e7f9 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,8 @@ +from typing import Optional + +import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponsesAPIRequestParams @@ -61,3 +65,44 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): extra_body=optional_params.get("extra_body"), timeout=optional_params.get("timeout"), ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + headers.update( + { + "Authorization": f"Bearer {api_key}", + } + ) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the endpoint for OpenAI responses API + """ + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + return f"{api_base}/responses" From c7e8534df43993f09aed4334f90ed8d81570dacb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:14:14 -0700 Subject: [PATCH 11/61] BaseResponsesAPIConfig --- litellm/llms/base_llm/responses/transformation.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 669558f0eb3..27b0a439c72 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -52,6 +52,16 @@ class BaseResponsesAPIConfig(ABC): pass + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + return {} + + @abstractmethod def get_complete_url( self, api_base: Optional[str], From 0f8de3d0a512d0bca9f730375b52777dc604cc38 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:33:26 -0700 Subject: [PATCH 12/61] add transform_request for OpenAI responses API --- .../llms/base_llm/responses/transformation.py | 31 +++++++++++-------- .../llms/openai/responses/transformation.py | 26 +++++++++++++--- litellm/types/llms/openai.py | 13 +++++--- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 27b0a439c72..76b0a5b329d 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -1,8 +1,13 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Optional, Union -from litellm.types.llms.openai import ResponsesAPIRequestParams +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIRequestParams, +) +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelInfo from ..chat.transformation import BaseLLMException @@ -48,7 +53,7 @@ class BaseResponsesAPIConfig(ABC): optional_params: dict, model: str, drop_params: bool, - ) -> ResponsesAPIRequestParams: + ) -> ResponsesAPIOptionalRequestParams: pass @@ -66,7 +71,6 @@ class BaseResponsesAPIConfig(ABC): self, api_base: Optional[str], model: str, - optional_params: dict, stream: Optional[bool] = None, ) -> str: """ @@ -80,15 +84,16 @@ class BaseResponsesAPIConfig(ABC): raise ValueError("api_base is required") return api_base - # @abstractmethod - # def transform_request( - # self, - # model: str, - # optional_params: dict, - # litellm_params: dict, - # headers: dict, - # ) -> dict: - # pass + @abstractmethod + def transform_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> ResponsesAPIRequestParams: + pass # @abstractmethod # def transform_response( diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 7c52679e7f9..d67846e1010 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,9 +1,14 @@ -from typing import Optional +from typing import Optional, Union import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ResponsesAPIRequestParams +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIRequestParams, +) +from litellm.types.router import GenericLiteLLMParams class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @@ -41,9 +46,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): optional_params: dict, model: str, drop_params: bool, - ) -> ResponsesAPIRequestParams: + ) -> ResponsesAPIOptionalRequestParams: - return ResponsesAPIRequestParams( + return ResponsesAPIOptionalRequestParams( include=optional_params.get("include"), instructions=optional_params.get("instructions"), max_output_tokens=optional_params.get("max_output_tokens"), @@ -66,6 +71,18 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): timeout=optional_params.get("timeout"), ) + def transform_request( + self, + model: str, + input: Union[str, ResponseInputParam], + response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> ResponsesAPIRequestParams: + return ResponsesAPIRequestParams( + model=model, input=input, **response_api_optional_request_params + ) + def validate_environment( self, headers: dict, @@ -89,7 +106,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): self, api_base: Optional[str], model: str, - optional_params: dict, stream: Optional[bool] = None, ) -> str: """ diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 9b6581773d6..1b521ece9d4 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -695,11 +695,9 @@ OpenAIAudioTranscriptionOptionalParams = Literal[ OpenAIImageVariationOptionalParams = Literal["n", "size", "response_format", "user"] -class ResponsesAPIRequestParams(TypedDict, total=False): - """TypedDict for parameters supported by the responses API.""" +class ResponsesAPIOptionalRequestParams(TypedDict, total=False): + """TypedDict for Optional parameters supported by the responses API.""" - input: Union[str, ResponseInputParam] - model: str include: Optional[List[ResponseIncludable]] instructions: Optional[str] max_output_tokens: Optional[int] @@ -720,3 +718,10 @@ class ResponsesAPIRequestParams(TypedDict, total=False): extra_query: Optional[Dict[str, Any]] extra_body: Optional[Dict[str, Any]] timeout: Optional[Union[float, httpx.Timeout]] + + +class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): + """TypedDict for request parameters supported by the responses API.""" + + input: Union[str, ResponseInputParam] + model: str From eafc1be132f30f93b65903459d8641e8b4e6635f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:46:28 -0700 Subject: [PATCH 13/61] add ResponsesAPIResponse --- litellm/types/llms/openai.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1b521ece9d4..1e57c3714b1 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -32,6 +32,16 @@ from openai.types.chat.chat_completion_prediction_content_param import ( ) from openai.types.embedding import Embedding as OpenAIEmbedding from openai.types.fine_tuning.fine_tuning_job import FineTuningJob +from openai.types.responses.response import ( + IncompleteDetails, + Reasoning, + Response, + ResponseOutputItem, + ResponseTextConfig, + ResponseUsage, + Tool, + ToolChoice, +) from openai.types.responses.response_create_params import ( Reasoning, ResponseIncludable, @@ -725,3 +735,28 @@ class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): input: Union[str, ResponseInputParam] model: str + + +class ResponsesAPIResponse(TypedDict, total=False): + id: str + created_at: float + error: Optional[dict] + incomplete_details: Optional[IncompleteDetails] + instructions: Optional[str] + metadata: Optional[Dict] + model: Optional[str] + object: Optional[str] + output: List[ResponseOutputItem] + parallel_tool_calls: bool + temperature: Optional[float] + tool_choice: ToolChoice + tools: List[Tool] + top_p: Optional[float] + max_output_tokens: Optional[int] + previous_response_id: Optional[str] + reasoning: Optional[Reasoning] + status: Optional[str] + text: Optional[ResponseTextConfig] + truncation: Optional[Literal["auto", "disabled"]] + usage: Optional[ResponseUsage] + user: Optional[str] From dff8308e922d4f61471a0b696fe6ae1a95495488 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:53:18 -0700 Subject: [PATCH 14/61] add transform_response_api_response --- .../llms/base_llm/responses/transformation.py | 19 +++++++++++++------ .../llms/openai/responses/transformation.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 76b0a5b329d..cc0a85b6f54 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -2,10 +2,13 @@ import types from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Optional, Union +import httpx + from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIRequestParams, + ResponsesAPIResponse, ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelInfo @@ -85,7 +88,7 @@ class BaseResponsesAPIConfig(ABC): return api_base @abstractmethod - def transform_request( + def transform_responses_api_request( self, model: str, input: Union[str, ResponseInputParam], @@ -95,8 +98,12 @@ class BaseResponsesAPIConfig(ABC): ) -> ResponsesAPIRequestParams: pass - # @abstractmethod - # def transform_response( - # self, - # ): - # pass + @abstractmethod + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ResponsesAPIResponse, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + return model_response diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index d67846e1010..fa4fe0274d8 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -71,7 +71,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): timeout=optional_params.get("timeout"), ) - def transform_request( + def transform_responses_api_request( self, model: str, input: Union[str, ResponseInputParam], From d3575f0a316089f5b2785982bf42504529d876a2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 16:58:43 -0700 Subject: [PATCH 15/61] working async_response_api_handler --- .../llms/base_llm/responses/transformation.py | 9 +++ litellm/llms/custom_httpx/llm_http_handler.py | 77 +++++++++++++++++-- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index cc0a85b6f54..03d496c0538 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -107,3 +107,12 @@ class BaseResponsesAPIConfig(ABC): logging_obj: LiteLLMLoggingObj, ) -> ResponsesAPIResponse: return model_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b8b7af686d3..fd64111f173 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -18,8 +18,14 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) -from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIRequestParams +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIRequestParams, + ResponsesAPIResponse, +) from litellm.types.rerank import OptionalRerankParams, RerankResponse +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager @@ -957,14 +963,75 @@ class BaseLLMHTTPHandler: async def async_response_api_handler( self, model: str, + custom_llm_provider: str, input: Union[str, ResponseInputParam], responses_api_provider_config: BaseResponsesAPIConfig, - responses_api_request_params: ResponsesAPIRequestParams, - ) -> Any: - pass + response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + logging_obj: LiteLLMLoggingObj, + litellm_params: GenericLiteLLMParams, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> ResponsesAPIResponse: + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider) + ) + else: + async_httpx_client = client + headers = responses_api_provider_config.validate_environment( + api_key=litellm_params.api_key, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + ) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + model=model, + ) + + data = responses_api_provider_config.transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=response_api_optional_request_params.get("timeout"), + ) + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + base_response_api_response = ResponsesAPIResponse() + return responses_api_provider_config.transform_response_api_response( + model=model, + raw_response=response, + model_response=base_response_api_response, + logging_obj=logging_obj, + ) def _handle_error( - self, e: Exception, provider_config: Union[BaseConfig, BaseRerankConfig] + self, + e: Exception, + provider_config: Union[BaseConfig, BaseRerankConfig, BaseResponsesAPIConfig], ): status_code = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) From b3999b4c75e6a66a66228b902f1248ccdb794706 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 17:03:31 -0700 Subject: [PATCH 16/61] transform_response_api_response --- .../llms/base_llm/responses/transformation.py | 3 +- .../llms/openai/responses/transformation.py | 28 ++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 03d496c0538..8bf7d68bc21 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -103,10 +103,9 @@ class BaseResponsesAPIConfig(ABC): self, model: str, raw_response: httpx.Response, - model_response: ResponsesAPIResponse, logging_obj: LiteLLMLoggingObj, ) -> ResponsesAPIResponse: - return model_response + pass def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index fa4fe0274d8..0e480e0c3c3 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,6 @@ -from typing import Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union + +import httpx import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -7,9 +9,19 @@ from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIRequestParams, + ResponsesAPIResponse, ) from litellm.types.router import GenericLiteLLMParams +from ..common_utils import OpenAIError + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def get_supported_openai_params(self, model: str) -> list: @@ -83,6 +95,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model=model, input=input, **response_api_optional_request_params ) + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + try: + raw_response_json = raw_response.json() + except Exception: + raise OpenAIError( + message=raw_response.text, status_code=raw_response.status_code + ) + return ResponsesAPIResponse(**raw_response_json) + def validate_environment( self, headers: dict, From 52b43f672b7cca227d1f137455f85adbfc9c200d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 17:35:43 -0700 Subject: [PATCH 17/61] working test_basic_openai_responses_api --- .../test_openai_responses_api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 67581cec008..dc4f9308ef2 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1,13 +1,18 @@ import os import sys +import pytest sys.path.insert(0, os.path.abspath("../..")) import litellm +import json -def test_basic_openai_responses_api(): - response = litellm.responses( +@pytest.mark.asyncio +async def test_basic_openai_responses_api(): + litellm._turn_on_debug() + response = await litellm.aresponses( model="gpt-4o", input="Tell me a three sentence bedtime story about a unicorn." ) + print("litellm response=", json.dumps(response, indent=4, default=str)) # validate_responses_api_response() From 2ac5aa24772b16531332fda45c242ffc8773beb9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 17:36:06 -0700 Subject: [PATCH 18/61] ResponsesAPIOptionalRequestParams --- litellm/types/llms/openai.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1e57c3714b1..4c0be0014dc 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -724,10 +724,6 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): top_p: Optional[float] truncation: Optional[Literal["auto", "disabled"]] user: Optional[str] - extra_headers: Optional[Dict[str, Any]] - extra_query: Optional[Dict[str, Any]] - extra_body: Optional[Dict[str, Any]] - timeout: Optional[Union[float, httpx.Timeout]] class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): From f32968409eee1aaefcfc899a5697b6f1e4140c5a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 17:37:19 -0700 Subject: [PATCH 19/61] working basic openai response api request --- .../llms/base_llm/responses/transformation.py | 8 +-- litellm/llms/custom_httpx/llm_http_handler.py | 9 ++-- .../llms/openai/responses/transformation.py | 32 ++---------- litellm/responses/main.py | 51 +++++++++++++++---- litellm/responses/utils.py | 17 ++++--- 5 files changed, 66 insertions(+), 51 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 8bf7d68bc21..ef82b25ddb8 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -1,6 +1,6 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union import httpx @@ -53,10 +53,10 @@ class BaseResponsesAPIConfig(ABC): @abstractmethod def map_openai_params( self, - optional_params: dict, + response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - ) -> ResponsesAPIOptionalRequestParams: + ) -> Dict: pass @@ -92,7 +92,7 @@ class BaseResponsesAPIConfig(ABC): self, model: str, input: Union[str, ResponseInputParam], - response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + response_api_optional_request_params: Dict, litellm_params: GenericLiteLLMParams, headers: dict, ) -> ResponsesAPIRequestParams: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index fd64111f173..6b7eda3f6c2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,6 +1,6 @@ import io import json -from typing import TYPE_CHECKING, Any, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union import httpx # type: ignore @@ -966,10 +966,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, input: Union[str, ResponseInputParam], responses_api_provider_config: BaseResponsesAPIConfig, - response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + response_api_optional_request_params: Dict, logging_obj: LiteLLMLoggingObj, litellm_params: GenericLiteLLMParams, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, ) -> ResponsesAPIResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( @@ -1020,11 +1023,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) - base_response_api_response = ResponsesAPIResponse() return responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, - model_response=base_response_api_response, logging_obj=logging_obj, ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 0e480e0c3c3..c30a4257908 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union import httpx @@ -55,39 +55,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def map_openai_params( self, - optional_params: dict, + response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - ) -> ResponsesAPIOptionalRequestParams: - - return ResponsesAPIOptionalRequestParams( - include=optional_params.get("include"), - instructions=optional_params.get("instructions"), - max_output_tokens=optional_params.get("max_output_tokens"), - metadata=optional_params.get("metadata"), - parallel_tool_calls=optional_params.get("parallel_tool_calls"), - previous_response_id=optional_params.get("previous_response_id"), - reasoning=optional_params.get("reasoning"), - store=optional_params.get("store"), - stream=optional_params.get("stream"), - temperature=optional_params.get("temperature"), - text=optional_params.get("text"), - tool_choice=optional_params.get("tool_choice"), - tools=optional_params.get("tools"), - top_p=optional_params.get("top_p"), - truncation=optional_params.get("truncation"), - user=optional_params.get("user"), - extra_headers=optional_params.get("extra_headers"), - extra_query=optional_params.get("extra_query"), - extra_body=optional_params.get("extra_body"), - timeout=optional_params.get("timeout"), - ) + ) -> Dict: + return dict(response_api_optional_params) def transform_responses_api_request( self, model: str, input: Union[str, ResponseInputParam], - response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + response_api_optional_request_params: Dict, litellm_params: GenericLiteLLMParams, headers: dict, ) -> ResponsesAPIRequestParams: diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 0920d25d170..817389d44e5 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,8 +1,9 @@ -from typing import Any, Dict, Iterable, List, Literal, Optional, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Union, get_type_hints import httpx import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.responses.utils import ( @@ -13,12 +14,13 @@ from litellm.types.llms.openai import ( Reasoning, ResponseIncludable, ResponseInputParam, + ResponsesAPIOptionalRequestParams, ResponseTextConfigParam, ToolChoice, ToolParam, ) from litellm.types.router import GenericLiteLLMParams -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, client ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here @@ -26,6 +28,24 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# +def get_requested_response_api_optional_param( + params: Dict[str, Any] +) -> ResponsesAPIOptionalRequestParams: + """ + Filter parameters to only include those defined in ResponsesAPIOptionalRequestParams. + + Args: + params: Dictionary of parameters to filter + + Returns: + ResponsesAPIOptionalRequestParams instance with only the valid parameters + """ + valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() + filtered_params = {k: v for k, v in params.items() if k in valid_keys} + return ResponsesAPIOptionalRequestParams(**filtered_params) + + +@client async def aresponses( input: Union[str, ResponseInputParam], model: str, @@ -53,6 +73,8 @@ async def aresponses( timeout: Optional[Union[float, httpx.Timeout]] = None, **kwargs, ): + litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) # get llm provider logic litellm_params = GenericLiteLLMParams(**kwargs) @@ -81,22 +103,31 @@ async def aresponses( ) # Get all parameters using locals() and combine with kwargs - all_params = {**locals(), **kwargs} + local_vars = locals() + local_vars.update(kwargs) + # Get ResponsesAPIOptionalRequestParams with only valid parameters + response_api_optional_params: ResponsesAPIOptionalRequestParams = ( + get_requested_response_api_optional_param(local_vars) + ) # Get optional parameters for the responses API - responses_api_request_params: ResponsesAPIRequestParams = ( - get_optional_params_responses_api( - model=model, - responses_api_provider_config=responses_api_provider_config, - optional_params={**locals(), **kwargs}, - ) + responses_api_request_params: Dict = get_optional_params_responses_api( + model=model, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_params=response_api_optional_params, ) response = await base_llm_http_handler.async_response_api_handler( model=model, input=input, responses_api_provider_config=responses_api_provider_config, - responses_api_request_params=responses_api_request_params, + response_api_optional_request_params=responses_api_request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, ) return response diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 51a77bc23c9..6d61cfe0826 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,15 +1,19 @@ +import json from typing import Any, Dict import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.types.llms.openai import ResponsesAPIRequestParams +from litellm.types.llms.openai import ( + ResponsesAPIOptionalRequestParams, + ResponsesAPIRequestParams, +) def get_optional_params_responses_api( model: str, responses_api_provider_config: BaseResponsesAPIConfig, - optional_params: Dict[str, Any], -) -> ResponsesAPIRequestParams: + response_api_optional_params: ResponsesAPIOptionalRequestParams, +) -> Dict: """ Get optional parameters for the responses API. @@ -22,14 +26,13 @@ def get_optional_params_responses_api( A dictionary of supported parameters for the responses API """ # Remove None values and internal parameters - filtered_params = {k: v for k, v in optional_params.items() if v is not None} # Get supported parameters for the model supported_params = responses_api_provider_config.get_supported_openai_params(model) # Check for unsupported parameters unsupported_params = [ - param for param in filtered_params if param not in supported_params + param for param in response_api_optional_params if param not in supported_params ] if unsupported_params: @@ -40,7 +43,9 @@ def get_optional_params_responses_api( # Map parameters to provider-specific format mapped_params = responses_api_provider_config.map_openai_params( - optional_params=filtered_params, model=model, drop_params=litellm.drop_params + response_api_optional_params=response_api_optional_params, + model=model, + drop_params=litellm.drop_params, ) return mapped_params From 8da714104b311b7ec2e30f0b649bb74e69a94455 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 17:48:15 -0700 Subject: [PATCH 20/61] ResponsesAPIStreamingResponse --- litellm/types/llms/openai.py | 5 +++++ .../test_openai_responses_api.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4c0be0014dc..f9e9a7a3344 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -756,3 +756,8 @@ class ResponsesAPIResponse(TypedDict, total=False): truncation: Optional[Literal["auto", "disabled"]] usage: Optional[ResponseUsage] user: Optional[str] + + +class ResponsesAPIStreamingResponse(TypedDict, total=False): + type: str + response: ResponsesAPIResponse diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index dc4f9308ef2..59024716d79 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -16,3 +16,16 @@ async def test_basic_openai_responses_api(): print("litellm response=", json.dumps(response, indent=4, default=str)) # validate_responses_api_response() + + +@pytest.mark.asyncio +async def test_basic_openai_responses_api_streaming(): + litellm._turn_on_debug() + response = await litellm.aresponses( + model="gpt-4o", + input="Tell me a three sentence bedtime story about a unicorn.", + stream=True, + ) + + for event in response: + print("litellm response=", json.dumps(event, indent=4, default=str)) From aa40cb5b261f3f87c8b8acc3f409890f33a674be Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 19:47:43 -0700 Subject: [PATCH 21/61] working ResponsesAPIStreamingIterator --- litellm/responses/streaming_iterator.py | 98 +++++++++++++++++++ .../test_openai_responses_api.py | 2 +- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 litellm/responses/streaming_iterator.py diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py new file mode 100644 index 00000000000..ad3fa59b26a --- /dev/null +++ b/litellm/responses/streaming_iterator.py @@ -0,0 +1,98 @@ +import json +from typing import Any, AsyncIterator, Dict, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.llms.openai import ( + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) + + +class ResponsesAPIStreamingIterator: + """ + Async iterator for processing streaming responses from the Responses API. + + This iterator handles the chunked streaming format returned by the Responses API + and yields properly formatted ResponsesAPIStreamingResponse objects. + """ + + def __init__( + self, + response: httpx.Response, + model: str, + logging_obj: Optional[LiteLLMLoggingObj] = None, + ): + self.response = response + self.model = model + self.logging_obj = logging_obj + self.stream_iterator = response.aiter_lines() + self.finished = False + + def __aiter__(self): + return self + + async def __anext__(self) -> ResponsesAPIStreamingResponse: + if self.finished: + raise StopAsyncIteration + + try: + # Get the next chunk from the stream + try: + chunk = await self.stream_iterator.__anext__() + except StopAsyncIteration: + self.finished = True + raise StopAsyncIteration + + if not chunk: + return await self.__anext__() + + # Handle SSE format (data: {...}) + if chunk.startswith("data: "): + chunk = chunk[6:] # Remove "data: " prefix + + # Handle "[DONE]" marker + if chunk == "[DONE]": + self.finished = True + raise StopAsyncIteration + + try: + # Parse the JSON chunk + parsed_chunk = json.loads(chunk) + + # Log the chunk if logging is enabled + if self.logging_obj: + self.logging_obj.post_call( + input="", + api_key="", + original_response=parsed_chunk, + additional_args={ + "complete_streaming_chunk": parsed_chunk, + }, + ) + + # Format as ResponsesAPIStreamingResponse + if isinstance(parsed_chunk, dict): + # If the chunk already has a 'type' field, it's already in the right format + if "type" in parsed_chunk: + return ResponsesAPIStreamingResponse(**parsed_chunk) + # Otherwise, wrap it as a response + else: + return ResponsesAPIStreamingResponse( + type="response", + response=ResponsesAPIResponse(**parsed_chunk), + ) + + return ResponsesAPIStreamingResponse( + type="response", response=parsed_chunk + ) + + except json.JSONDecodeError: + # If we can't parse the chunk, continue to the next one + return await self.__anext__() + + except httpx.HTTPError as e: + # Handle HTTP errors + self.finished = True + raise e diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 59024716d79..54464c05099 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -27,5 +27,5 @@ async def test_basic_openai_responses_api_streaming(): stream=True, ) - for event in response: + async for event in response: print("litellm response=", json.dumps(event, indent=4, default=str)) From cb270887a7b3ac7b4ceb08f53aad9ecbf9437998 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 19:49:09 -0700 Subject: [PATCH 22/61] ResponsesAPIStreamingIterator --- litellm/llms/custom_httpx/llm_http_handler.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6b7eda3f6c2..5ce05f4da75 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -18,6 +18,7 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) +from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, @@ -973,7 +974,7 @@ class BaseLLMHTTPHandler: extra_headers: Optional[Dict[str, Any]] = None, extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> ResponsesAPIResponse: + ) -> Union[ResponsesAPIResponse, ResponsesAPIStreamingIterator]: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider) @@ -1010,13 +1011,34 @@ class BaseLLMHTTPHandler: }, ) + # Check if streaming is requested + stream = response_api_optional_request_params.get("stream", False) + try: - response = await async_httpx_client.post( - url=api_base, - headers=headers, - data=json.dumps(data), - timeout=response_api_optional_request_params.get("timeout"), - ) + if stream: + # For streaming, we need to use stream=True in the request + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=response_api_optional_request_params.get("timeout"), + stream=True, + ) + + # Return the streaming iterator + return ResponsesAPIStreamingIterator( + response=response, + model=model, + logging_obj=logging_obj, + ) + else: + # For non-streaming, proceed as before + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=response_api_optional_request_params.get("timeout"), + ) except Exception as e: raise self._handle_error( e=e, From 8fa313ab07d86f0849a905206634f61c1a968f9b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 20:00:42 -0700 Subject: [PATCH 23/61] add async streaming support --- .../llms/base_llm/responses/transformation.py | 13 +++++++ litellm/llms/custom_httpx/llm_http_handler.py | 1 + .../llms/openai/responses/transformation.py | 13 +++++++ litellm/responses/streaming_iterator.py | 37 ++++++++----------- 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index ef82b25ddb8..bca8e3c7ed5 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -9,6 +9,7 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIRequestParams, ResponsesAPIResponse, + ResponsesAPIStreamingResponse, ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelInfo @@ -107,6 +108,18 @@ class BaseResponsesAPIConfig(ABC): ) -> ResponsesAPIResponse: pass + @abstractmethod + def transform_streaming_response( + self, + model: str, + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIStreamingResponse: + """ + Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse + """ + pass + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 5ce05f4da75..a303c4572cb 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1030,6 +1030,7 @@ class BaseLLMHTTPHandler: response=response, model=model, logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, ) else: # For non-streaming, proceed as before diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index c30a4257908..cecc9efe765 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -10,6 +10,7 @@ from litellm.types.llms.openai import ( ResponsesAPIOptionalRequestParams, ResponsesAPIRequestParams, ResponsesAPIResponse, + ResponsesAPIStreamingResponse, ) from litellm.types.router import GenericLiteLLMParams @@ -126,3 +127,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): api_base = api_base.rstrip("/") return f"{api_base}/responses" + + def transform_streaming_response( + self, + model: str, + parsed_chunk: dict, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIStreamingResponse: + """ + Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse + """ + # Convert the dictionary to a properly typed ResponsesAPIStreamingResponse + return ResponsesAPIStreamingResponse(**parsed_chunk) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index ad3fa59b26a..ae592d8c411 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -4,10 +4,12 @@ from typing import Any, AsyncIterator, Dict, Optional, Union import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamingResponse, ) +from litellm.utils import CustomStreamWrapper class ResponsesAPIStreamingIterator: @@ -22,13 +24,15 @@ class ResponsesAPIStreamingIterator: self, response: httpx.Response, model: str, - logging_obj: Optional[LiteLLMLoggingObj] = None, + responses_api_provider_config: BaseResponsesAPIConfig, + logging_obj: LiteLLMLoggingObj, ): self.response = response self.model = model self.logging_obj = logging_obj self.stream_iterator = response.aiter_lines() self.finished = False + self.responses_api_provider_config = responses_api_provider_config def __aiter__(self): return self @@ -49,8 +53,9 @@ class ResponsesAPIStreamingIterator: return await self.__anext__() # Handle SSE format (data: {...}) - if chunk.startswith("data: "): - chunk = chunk[6:] # Remove "data: " prefix + chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) + if chunk is None: + return await self.__anext__() # Handle "[DONE]" marker if chunk == "[DONE]": @@ -61,28 +66,16 @@ class ResponsesAPIStreamingIterator: # Parse the JSON chunk parsed_chunk = json.loads(chunk) - # Log the chunk if logging is enabled - if self.logging_obj: - self.logging_obj.post_call( - input="", - api_key="", - original_response=parsed_chunk, - additional_args={ - "complete_streaming_chunk": parsed_chunk, - }, - ) - # Format as ResponsesAPIStreamingResponse if isinstance(parsed_chunk, dict): - # If the chunk already has a 'type' field, it's already in the right format - if "type" in parsed_chunk: - return ResponsesAPIStreamingResponse(**parsed_chunk) - # Otherwise, wrap it as a response - else: - return ResponsesAPIStreamingResponse( - type="response", - response=ResponsesAPIResponse(**parsed_chunk), + openai_responses_api_chunk: ResponsesAPIStreamingResponse = ( + self.responses_api_provider_config.transform_streaming_response( + model=self.model, + parsed_chunk=parsed_chunk, + logging_obj=self.logging_obj, ) + ) + return openai_responses_api_chunk return ResponsesAPIStreamingResponse( type="response", response=parsed_chunk From 24cb83b0e4dcd59a5ad7e44c2a95f9b096019bfd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:02:14 -0700 Subject: [PATCH 24/61] Response API cost tracking --- litellm/litellm_core_utils/litellm_logging.py | 9 +++++ litellm/responses/utils.py | 23 +++++++++++ litellm/types/llms/openai.py | 38 ++++++++++++++++--- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a3d9a57a497..7adecc511d1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -39,11 +39,13 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, ) +from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( AllMessageValues, Batch, FineTuningJob, HttpxBinaryResponseContent, + ResponsesAPIResponse, ) from litellm.types.rerank import RerankResponse from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS @@ -851,6 +853,7 @@ class Logging(LiteLLMLoggingBaseClass): RerankResponse, Batch, FineTuningJob, + ResponsesAPIResponse, ], cache_hit: Optional[bool] = None, ) -> Optional[float]: @@ -3111,6 +3114,12 @@ class StandardLoggingPayloadSetup: elif isinstance(usage, Usage): return usage elif isinstance(usage, dict): + if ResponseAPILoggingUtils._is_response_api_usage(usage): + return ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + ) return Usage(**usage) raise ValueError(f"usage is required, got={usage} of type {type(usage)}") diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 6d61cfe0826..c2775d9f579 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -4,9 +4,11 @@ from typing import Any, Dict import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( + ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIRequestParams, ) +from litellm.types.utils import Usage def get_optional_params_responses_api( @@ -49,3 +51,24 @@ def get_optional_params_responses_api( ) return mapped_params + + +class ResponseAPILoggingUtils: + @staticmethod + def _is_response_api_usage(usage: dict) -> bool: + """returns True if usage is from OpenAI Response API""" + if "input_tokens" in usage and "output_tokens" in usage: + return True + return False + + @staticmethod + def _transform_response_api_usage_to_chat_usage(usage: dict) -> Usage: + """Tranforms the ResponseAPIUsage object to a Usage object""" + response_api_usage: ResponseAPIUsage = ResponseAPIUsage(**usage) + prompt_tokens: int = response_api_usage.input_tokens or 0 + completion_tokens: int = response_api_usage.output_tokens or 0 + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index f9e9a7a3344..55f05b5cdca 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -38,7 +38,6 @@ from openai.types.responses.response import ( Response, ResponseOutputItem, ResponseTextConfig, - ResponseUsage, Tool, ToolChoice, ) @@ -50,7 +49,7 @@ from openai.types.responses.response_create_params import ( ToolChoice, ToolParam, ) -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, PrivateAttr from typing_extensions import Dict, Required, TypedDict, override FileContent = Union[IO[bytes], bytes, PathLike] @@ -733,7 +732,25 @@ class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): model: str -class ResponsesAPIResponse(TypedDict, total=False): +class OutputTokensDetails(BaseModel): + reasoning_tokens: int + + +class ResponseAPIUsage(BaseModel): + input_tokens: int + """The number of input tokens.""" + + output_tokens: int + """The number of output tokens.""" + + output_tokens_details: OutputTokensDetails + """A detailed breakdown of the output tokens.""" + + total_tokens: int + """The total number of tokens used.""" + + +class ResponsesAPIResponse(BaseModel): id: str created_at: float error: Optional[dict] @@ -754,10 +771,21 @@ class ResponsesAPIResponse(TypedDict, total=False): status: Optional[str] text: Optional[ResponseTextConfig] truncation: Optional[Literal["auto", "disabled"]] - usage: Optional[ResponseUsage] + usage: Optional[ResponseAPIUsage] user: Optional[str] + # Define private attributes using PrivateAttr + _hidden_params: dict = PrivateAttr(default_factory=dict) + + def __getitem__(self, key): + return self.__dict__[key] + + def get(self, key, default=None): + return self.__dict__.get(key, default) + + def __contains__(self, key): + return key in self.__dict__ -class ResponsesAPIStreamingResponse(TypedDict, total=False): +class ResponsesAPIStreamingResponse(BaseModel): type: str response: ResponsesAPIResponse From 8ada5a469dd9f4622e94ac962aa27ef56b9d1024 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:02:48 -0700 Subject: [PATCH 25/61] add responses api to call types --- litellm/types/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4af88100faf..04b76f3a2e4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -191,6 +191,8 @@ class CallTypes(Enum): retrieve_batch = "retrieve_batch" pass_through = "pass_through_endpoint" anthropic_messages = "anthropic_messages" + responses = "responses" + aresponses = "aresponses" CallTypesLiteral = Literal[ From 20e33984763c545b7938d8caa6a96abe51bf1f05 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:03:48 -0700 Subject: [PATCH 26/61] fix typing for aresponses --- litellm/responses/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 817389d44e5..c719b993a9c 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -15,6 +15,7 @@ from litellm.types.llms.openai import ( ResponseIncludable, ResponseInputParam, ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, ResponseTextConfigParam, ToolChoice, ToolParam, @@ -22,6 +23,8 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client +from .streaming_iterator import ResponsesAPIStreamingIterator + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -72,7 +75,7 @@ async def aresponses( extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, **kwargs, -): +) -> Union[ResponsesAPIResponse, ResponsesAPIStreamingIterator]: litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) @@ -131,8 +134,6 @@ async def aresponses( ) return response - pass - def responses( input: Union[str, ResponseInputParam], From ddb819da4583a56f3ac63335ba73df28ce7220fa Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:04:24 -0700 Subject: [PATCH 27/61] fix order of imports --- litellm/llms/base_llm/responses/transformation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index bca8e3c7ed5..6e27fd7a542 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -14,14 +14,16 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelInfo -from ..chat.transformation import BaseLLMException - if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from ..chat.transformation import BaseLLMException as _BaseLLMException + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException else: LiteLLMLoggingObj = Any + BaseLLMException = Any class BaseResponsesAPIConfig(ABC): @@ -123,6 +125,8 @@ class BaseResponsesAPIConfig(ABC): def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: + from ..chat.transformation import BaseLLMException + raise BaseLLMException( status_code=status_code, message=error_message, From 51dc24a40548c3d7856a888410848e444072b1dc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:26:44 -0700 Subject: [PATCH 28/61] _transform_response_api_usage_to_chat_usage --- litellm/cost_calculator.py | 12 ++- litellm/litellm_core_utils/litellm_logging.py | 1 + litellm/responses/streaming_iterator.py | 22 ++++- litellm/types/llms/openai.py | 6 +- .../test_openai_responses_api.py | 88 +++++++++++++++++++ 5 files changed, 124 insertions(+), 5 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 1d10fa1f9e1..fb49c079812 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -44,7 +44,8 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_ai_image_cost_calculator, ) -from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( CallTypesLiteral, @@ -601,6 +602,14 @@ def completion_cost( # noqa: PLR0915 _usage = usage_obj.model_dump() else: _usage = usage_obj + + if ResponseAPILoggingUtils._is_response_api_usage(_usage): + _usage = ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + _usage + ).model_dump() + ) + # get input/output tokens from completion_response prompt_tokens = _usage.get("prompt_tokens", 0) completion_tokens = _usage.get("completion_tokens", 0) @@ -799,6 +808,7 @@ def response_cost_calculator( TextCompletionResponse, HttpxBinaryResponseContent, RerankResponse, + ResponsesAPIResponse, ], model: str, custom_llm_provider: Optional[str], diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7adecc511d1..8a07ac9a8eb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1015,6 +1015,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(result, RerankResponse) or isinstance(result, FineTuningJob) or isinstance(result, LiteLLMBatch) + or isinstance(result, ResponsesAPIResponse) ): ## HIDDEN PARAMS ## hidden_params = getattr(result, "_hidden_params", {}) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index ae592d8c411..b99d81309a5 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1,4 +1,6 @@ +import asyncio import json +from datetime import datetime from typing import Any, AsyncIterator, Dict, Optional, Union import httpx @@ -11,6 +13,8 @@ from litellm.types.llms.openai import ( ) from litellm.utils import CustomStreamWrapper +COMPLETED_OPENAI_CHUNK_TYPE = "response.completed" + class ResponsesAPIStreamingIterator: """ @@ -33,14 +37,13 @@ class ResponsesAPIStreamingIterator: self.stream_iterator = response.aiter_lines() self.finished = False self.responses_api_provider_config = responses_api_provider_config + self.completed_response: Optional[ResponsesAPIStreamingResponse] = None + self.start_time = datetime.now() def __aiter__(self): return self async def __anext__(self) -> ResponsesAPIStreamingResponse: - if self.finished: - raise StopAsyncIteration - try: # Get the next chunk from the stream try: @@ -75,6 +78,19 @@ class ResponsesAPIStreamingIterator: logging_obj=self.logging_obj, ) ) + # Store the completed response + if ( + openai_responses_api_chunk + and openai_responses_api_chunk.type + == COMPLETED_OPENAI_CHUNK_TYPE + ): + self.completed_response = openai_responses_api_chunk + await self.logging_obj.async_success_handler( + result=self.completed_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) return openai_responses_api_chunk return ResponsesAPIStreamingResponse( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 55f05b5cdca..ddca244b467 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -735,6 +735,8 @@ class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): class OutputTokensDetails(BaseModel): reasoning_tokens: int + model_config = {"extra": "allow"} + class ResponseAPIUsage(BaseModel): input_tokens: int @@ -743,12 +745,14 @@ class ResponseAPIUsage(BaseModel): output_tokens: int """The number of output tokens.""" - output_tokens_details: OutputTokensDetails + output_tokens_details: Optional[OutputTokensDetails] """A detailed breakdown of the output tokens.""" total_tokens: int """The total number of tokens used.""" + model_config = {"extra": "allow"} + class ResponsesAPIResponse(BaseModel): id: str diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 54464c05099..32e2e4f80f6 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1,10 +1,14 @@ import os import sys import pytest +import asyncio +from typing import Optional sys.path.insert(0, os.path.abspath("../..")) import litellm +from litellm.integrations.custom_logger import CustomLogger import json +from litellm.types.utils import StandardLoggingPayload @pytest.mark.asyncio @@ -29,3 +33,87 @@ async def test_basic_openai_responses_api_streaming(): async for event in response: print("litellm response=", json.dumps(event, indent=4, default=str)) + + +class TestCustomLogger(CustomLogger): + def __init__( + self, + ): + self.standard_logging_object: Optional[StandardLoggingPayload] = None + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + print("in async_log_success_event") + self.standard_logging_object = kwargs["standard_logging_object"] + pass + + +@pytest.mark.asyncio +async def test_basic_openai_responses_api_streaming_with_logging(): + litellm._turn_on_debug() + litellm.set_verbose = True + test_custom_logger = TestCustomLogger() + litellm.callbacks = [test_custom_logger] + response = await litellm.aresponses( + model="gpt-4o", + input="hi", + stream=True, + ) + + async for event in response: + print("litellm response=", json.dumps(event, indent=4, default=str)) + + print("sleeping for 2 seconds...") + await asyncio.sleep(2) + print( + "standard logging payload=", + json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), + ) + + +@pytest.mark.asyncio +async def test_basic_openai_responses_api_non_streaming_with_logging(): + litellm._turn_on_debug() + litellm.set_verbose = True + test_custom_logger = TestCustomLogger() + litellm.callbacks = [test_custom_logger] + response = await litellm.aresponses( + model="gpt-4o", + input="hi", + ) + + print("litellm response=", json.dumps(response, indent=4, default=str)) + print("response hidden params=", response._hidden_params) + + print("sleeping for 2 seconds...") + await asyncio.sleep(2) + print( + "standard logging payload=", + json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), + ) + + assert test_custom_logger.standard_logging_object is not None + + # validate token counts match OpenAI response + assert ( + test_custom_logger.standard_logging_object["prompt_tokens"] + == response["usage"]["input_tokens"] + ) + assert ( + test_custom_logger.standard_logging_object["completion_tokens"] + == response["usage"]["output_tokens"] + ) + assert ( + test_custom_logger.standard_logging_object["total_tokens"] + == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] + ) + + # validate spend > 0 + assert test_custom_logger.standard_logging_object["response_cost"] > 0 + + # validate response id matches OpenAI + + # validate model matches + + # validate messages matches + + # validate responses matches From b790f0a5c61edba4ba6ed55dcbe7df0c87ac2cd2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:34:18 -0700 Subject: [PATCH 29/61] log input of response API --- litellm/responses/main.py | 12 ++++++++++++ litellm/utils.py | 5 +++++ .../test_openai_responses_api.py | 8 +++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c719b993a9c..337e7fc3b07 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -120,6 +120,18 @@ async def aresponses( response_api_optional_params=response_api_optional_params, ) + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model=model, + user=user, + optional_params=dict(responses_api_request_params), + litellm_params={ + "litellm_call_id": litellm_call_id, + **responses_api_request_params, + }, + custom_llm_provider=custom_llm_provider, + ) + response = await base_llm_http_handler.async_response_api_handler( model=model, input=input, diff --git a/litellm/utils.py b/litellm/utils.py index 97c8171b362..cebc0ed7cf3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -716,6 +716,11 @@ def function_setup( # noqa: PLR0915 call_type == CallTypes.aspeech.value or call_type == CallTypes.speech.value ): messages = kwargs.get("input", "speech") + elif ( + call_type == CallTypes.aresponses.value + or call_type == CallTypes.responses.value + ): + messages = args[0] if len(args) > 0 else kwargs["input"] else: messages = "default-message-value" stream = True if "stream" in kwargs and kwargs["stream"] is True else False diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 32e2e4f80f6..6d86cddb762 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -76,8 +76,9 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): litellm.set_verbose = True test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] + request_model = "gpt-4o" response = await litellm.aresponses( - model="gpt-4o", + model=request_model, input="hi", ) @@ -111,9 +112,14 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): assert test_custom_logger.standard_logging_object["response_cost"] > 0 # validate response id matches OpenAI + assert test_custom_logger.standard_logging_object["id"] == response["id"] # validate model matches + assert test_custom_logger.standard_logging_object["model"] == request_model # validate messages matches + assert test_custom_logger.standard_logging_object["messages"] == [ + {"content": "hi", "role": "user"} + ] # validate responses matches From dd7ac41e3305b9cff79a0560d366317b5fc97da7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 22:42:49 -0700 Subject: [PATCH 30/61] validate_responses_match --- .../test_openai_responses_api.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 6d86cddb762..efa6dcf4055 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -70,6 +70,43 @@ async def test_basic_openai_responses_api_streaming_with_logging(): ) +def validate_responses_match(slp_response, litellm_response): + """Validate that the standard logging payload OpenAI response matches the litellm response""" + # Validate core fields + assert slp_response["id"] == litellm_response["id"], "ID mismatch" + assert slp_response["model"] == litellm_response["model"], "Model mismatch" + assert ( + slp_response["created_at"] == litellm_response["created_at"] + ), "Created at mismatch" + + # Validate usage + assert ( + slp_response["usage"]["input_tokens"] + == litellm_response["usage"]["input_tokens"] + ), "Input tokens mismatch" + assert ( + slp_response["usage"]["output_tokens"] + == litellm_response["usage"]["output_tokens"] + ), "Output tokens mismatch" + assert ( + slp_response["usage"]["total_tokens"] + == litellm_response["usage"]["total_tokens"] + ), "Total tokens mismatch" + + # Validate output/messages + assert len(slp_response["output"]) == len( + litellm_response["output"] + ), "Output length mismatch" + for slp_msg, litellm_msg in zip(slp_response["output"], litellm_response["output"]): + assert slp_msg["role"] == litellm_msg.role, "Message role mismatch" + # Access the content's text field for the litellm response + litellm_content = litellm_msg.content[0].text if litellm_msg.content else "" + assert ( + slp_msg["content"][0]["text"] == litellm_content + ), f"Message content mismatch. Expected {litellm_content}, Got {slp_msg['content']}" + assert slp_msg["status"] == litellm_msg.status, "Message status mismatch" + + @pytest.mark.asyncio async def test_basic_openai_responses_api_non_streaming_with_logging(): litellm._turn_on_debug() @@ -122,4 +159,7 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): {"content": "hi", "role": "user"} ] - # validate responses matches + # Add validation after existing assertions + validate_responses_match( + test_custom_logger.standard_logging_object["response"], response + ) From 278b6fb5f6531e09373d37d83fb2ec0a949f6b6a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 23:13:10 -0700 Subject: [PATCH 31/61] add debug logging --- litellm/llms/openai/responses/transformation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index cecc9efe765..b6ba970ff47 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union import httpx import litellm +from litellm._logging import verbose_logger from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -138,4 +139,5 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse """ # Convert the dictionary to a properly typed ResponsesAPIStreamingResponse + verbose_logger.debug("Raw OpenAI Chunk=%s", parsed_chunk) return ResponsesAPIStreamingResponse(**parsed_chunk) From 4ff6e41c15ebbc04a39dfaaeed133fdf46e00069 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Mar 2025 23:42:35 -0700 Subject: [PATCH 32/61] ResponsesAPIStreamEvents --- .../llms/openai/responses/transformation.py | 62 ++++- litellm/types/llms/openai.py | 247 +++++++++++++++++- 2 files changed, 296 insertions(+), 13 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b6ba970ff47..84641829db5 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast import httpx @@ -6,13 +6,7 @@ import litellm from litellm._logging import verbose_logger from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ( - ResponseInputParam, - ResponsesAPIOptionalRequestParams, - ResponsesAPIRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamingResponse, -) +from litellm.types.llms.openai import * from litellm.types.router import GenericLiteLLMParams from ..common_utils import OpenAIError @@ -140,4 +134,54 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ # Convert the dictionary to a properly typed ResponsesAPIStreamingResponse verbose_logger.debug("Raw OpenAI Chunk=%s", parsed_chunk) - return ResponsesAPIStreamingResponse(**parsed_chunk) + event_type = str(parsed_chunk.get("type")) + event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class( + event_type=event_type + ) + return event_pydantic_model(**parsed_chunk) + + @staticmethod + def get_event_model_class(event_type: str) -> Any: + """ + Returns the appropriate event model class based on the event type. + + Args: + event_type (str): The type of event from the response chunk + + Returns: + Any: The corresponding event model class + + Raises: + ValueError: If the event type is unknown + """ + event_models = { + ResponsesAPIStreamEvents.RESPONSE_CREATED: ResponseCreatedEvent, + ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS: ResponseInProgressEvent, + ResponsesAPIStreamEvents.RESPONSE_COMPLETED: ResponseCompletedEvent, + ResponsesAPIStreamEvents.RESPONSE_FAILED: ResponseFailedEvent, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE: ResponseIncompleteEvent, + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED: OutputItemAddedEvent, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: OutputItemDoneEvent, + ResponsesAPIStreamEvents.CONTENT_PART_ADDED: ContentPartAddedEvent, + ResponsesAPIStreamEvents.CONTENT_PART_DONE: ContentPartDoneEvent, + ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA: OutputTextDeltaEvent, + ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED: OutputTextAnnotationAddedEvent, + ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: OutputTextDoneEvent, + ResponsesAPIStreamEvents.REFUSAL_DELTA: RefusalDeltaEvent, + ResponsesAPIStreamEvents.REFUSAL_DONE: RefusalDoneEvent, + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA: FunctionCallArgumentsDeltaEvent, + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE: FunctionCallArgumentsDoneEvent, + ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS: FileSearchCallInProgressEvent, + ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING: FileSearchCallSearchingEvent, + ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED: FileSearchCallCompletedEvent, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS: WebSearchCallInProgressEvent, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING: WebSearchCallSearchingEvent, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED: WebSearchCallCompletedEvent, + ResponsesAPIStreamEvents.ERROR: ErrorEvent, + } + + model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type)) + if not model_class: + raise ValueError(f"Unknown event type: {event_type}") + + return model_class diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index ddca244b467..5d6adc263ed 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,5 +1,17 @@ +from enum import Enum from os import PathLike -from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union +from typing import ( + IO, + Annotated, + Any, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import httpx from openai._legacy_response import ( @@ -49,7 +61,7 @@ from openai.types.responses.response_create_params import ( ToolChoice, ToolParam, ) -from pydantic import BaseModel, Field, PrivateAttr +from pydantic import BaseModel, Discriminator, Field, PrivateAttr from typing_extensions import Dict, Required, TypedDict, override FileContent = Union[IO[bytes], bytes, PathLike] @@ -790,6 +802,233 @@ class ResponsesAPIResponse(BaseModel): return key in self.__dict__ -class ResponsesAPIStreamingResponse(BaseModel): - type: str +class ResponsesAPIStreamEvents(str, Enum): + """ + Enum representing all supported OpenAI stream event types for the Responses API. + + Inherits from str to allow direct string comparison and usage as dictionary keys. + """ + + # Response lifecycle events + RESPONSE_CREATED = "response.created" + RESPONSE_IN_PROGRESS = "response.in_progress" + RESPONSE_COMPLETED = "response.completed" + RESPONSE_FAILED = "response.failed" + RESPONSE_INCOMPLETE = "response.incomplete" + + # Output item events + OUTPUT_ITEM_ADDED = "response.output_item.added" + OUTPUT_ITEM_DONE = "response.output_item.done" + + # Content part events + CONTENT_PART_ADDED = "response.content_part.added" + CONTENT_PART_DONE = "response.content_part.done" + + # Output text events + OUTPUT_TEXT_DELTA = "response.output_text.delta" + OUTPUT_TEXT_ANNOTATION_ADDED = "response.output_text.annotation.added" + OUTPUT_TEXT_DONE = "response.output_text.done" + + # Refusal events + REFUSAL_DELTA = "response.refusal.delta" + REFUSAL_DONE = "response.refusal.done" + + # Function call events + FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" + FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" + + # File search events + FILE_SEARCH_CALL_IN_PROGRESS = "response.file_search_call.in_progress" + FILE_SEARCH_CALL_SEARCHING = "response.file_search_call.searching" + FILE_SEARCH_CALL_COMPLETED = "response.file_search_call.completed" + + # Web search events + WEB_SEARCH_CALL_IN_PROGRESS = "response.web_search_call.in_progress" + WEB_SEARCH_CALL_SEARCHING = "response.web_search_call.searching" + WEB_SEARCH_CALL_COMPLETED = "response.web_search_call.completed" + + # Error event + ERROR = "error" + + +# Base streaming response types +class ResponseCreatedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED] response: ResponsesAPIResponse + + +class ResponseInProgressEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS] + response: ResponsesAPIResponse + + +class ResponseCompletedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + response: ResponsesAPIResponse + + +class ResponseFailedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_FAILED] + response: ResponsesAPIResponse + + +class ResponseIncompleteEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] + response: ResponsesAPIResponse + + +class OutputItemAddedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] + output_index: int + item: dict + + +class OutputItemDoneEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] + output_index: int + item: dict + + +class ContentPartAddedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] + item_id: str + output_index: int + content_index: int + part: dict + + +class ContentPartDoneEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] + item_id: str + output_index: int + content_index: int + part: dict + + +class OutputTextDeltaEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] + item_id: str + output_index: int + content_index: int + delta: str + + +class OutputTextAnnotationAddedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED] + item_id: str + output_index: int + content_index: int + annotation_index: int + annotation: dict + + +class OutputTextDoneEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] + item_id: str + output_index: int + content_index: int + text: str + + +class RefusalDeltaEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.REFUSAL_DELTA] + item_id: str + output_index: int + content_index: int + delta: str + + +class RefusalDoneEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.REFUSAL_DONE] + item_id: str + output_index: int + content_index: int + refusal: str + + +class FunctionCallArgumentsDeltaEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] + item_id: str + output_index: int + delta: str + + +class FunctionCallArgumentsDoneEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE] + item_id: str + output_index: int + arguments: str + + +class FileSearchCallInProgressEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS] + output_index: int + item_id: str + + +class FileSearchCallSearchingEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING] + output_index: int + item_id: str + + +class FileSearchCallCompletedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED] + output_index: int + item_id: str + + +class WebSearchCallInProgressEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS] + output_index: int + item_id: str + + +class WebSearchCallSearchingEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING] + output_index: int + item_id: str + + +class WebSearchCallCompletedEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED] + output_index: int + item_id: str + + +class ErrorEvent(BaseModel): + type: Literal[ResponsesAPIStreamEvents.ERROR] + code: Optional[str] + message: str + param: Optional[str] + + +# Union type for all possible streaming responses +ResponsesAPIStreamingResponse = Annotated[ + Union[ + ResponseCreatedEvent, + ResponseInProgressEvent, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + OutputItemAddedEvent, + OutputItemDoneEvent, + ContentPartAddedEvent, + ContentPartDoneEvent, + OutputTextDeltaEvent, + OutputTextAnnotationAddedEvent, + OutputTextDoneEvent, + RefusalDeltaEvent, + RefusalDoneEvent, + FunctionCallArgumentsDeltaEvent, + FunctionCallArgumentsDoneEvent, + FileSearchCallInProgressEvent, + FileSearchCallSearchingEvent, + FileSearchCallCompletedEvent, + WebSearchCallInProgressEvent, + WebSearchCallSearchingEvent, + WebSearchCallCompletedEvent, + ErrorEvent, + ], + Discriminator("type"), +] From fde75a068afd9cf520a4b71a211c4ace96946f9d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 00:02:39 -0700 Subject: [PATCH 33/61] working streaming logging --- litellm/litellm_core_utils/litellm_logging.py | 7 ++- litellm/responses/streaming_iterator.py | 20 +++---- litellm/types/llms/openai.py | 58 +++++++++++-------- 3 files changed, 49 insertions(+), 36 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8a07ac9a8eb..964bbfb70cd 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -45,6 +45,7 @@ from litellm.types.llms.openai import ( Batch, FineTuningJob, HttpxBinaryResponseContent, + ResponseCompletedEvent, ResponsesAPIResponse, ) from litellm.types.rerank import RerankResponse @@ -854,6 +855,7 @@ class Logging(LiteLLMLoggingBaseClass): Batch, FineTuningJob, ResponsesAPIResponse, + ResponseCompletedEvent, ], cache_hit: Optional[bool] = None, ) -> Optional[float]: @@ -1000,9 +1002,7 @@ class Logging(LiteLLMLoggingBaseClass): ## if model in model cost map - log the response cost ## else set cost to None if ( - standard_logging_object is None - and result is not None - and self.stream is not True + standard_logging_object is None and result is not None ): # handle streaming separately if ( isinstance(result, ModelResponse) @@ -1016,6 +1016,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(result, FineTuningJob) or isinstance(result, LiteLLMBatch) or isinstance(result, ResponsesAPIResponse) + or isinstance(result, ResponseCompletedEvent) ): ## HIDDEN PARAMS ## hidden_params = getattr(result, "_hidden_params", {}) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index b99d81309a5..7464d04607d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( ResponsesAPIResponse, + ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) from litellm.utils import CustomStreamWrapper @@ -82,21 +83,20 @@ class ResponsesAPIStreamingIterator: if ( openai_responses_api_chunk and openai_responses_api_chunk.type - == COMPLETED_OPENAI_CHUNK_TYPE + == ResponsesAPIStreamEvents.RESPONSE_COMPLETED ): self.completed_response = openai_responses_api_chunk - await self.logging_obj.async_success_handler( - result=self.completed_response, - start_time=self.start_time, - end_time=datetime.now(), - cache_hit=None, + asyncio.create_task( + self.logging_obj.async_success_handler( + result=self.completed_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) ) return openai_responses_api_chunk - return ResponsesAPIStreamingResponse( - type="response", response=parsed_chunk - ) - + return await self.__anext__() except json.JSONDecodeError: # If we can't parse the chunk, continue to the next one return await self.__anext__() diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 5d6adc263ed..9e55ac30e99 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -852,44 +852,56 @@ class ResponsesAPIStreamEvents(str, Enum): # Base streaming response types -class ResponseCreatedEvent(BaseModel): +class BaseResponseAPIStreamEvent(BaseModel): + def __getitem__(self, key): + return self.__dict__[key] + + def get(self, key, default=None): + return self.__dict__.get(key, default) + + def __contains__(self, key): + return key in self.__dict__ + + +class ResponseCreatedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED] response: ResponsesAPIResponse -class ResponseInProgressEvent(BaseModel): +class ResponseInProgressEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS] response: ResponsesAPIResponse -class ResponseCompletedEvent(BaseModel): +class ResponseCompletedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED] response: ResponsesAPIResponse + _hidden_params: dict = PrivateAttr(default_factory=dict) -class ResponseFailedEvent(BaseModel): +class ResponseFailedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_FAILED] response: ResponsesAPIResponse -class ResponseIncompleteEvent(BaseModel): +class ResponseIncompleteEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] response: ResponsesAPIResponse -class OutputItemAddedEvent(BaseModel): +class OutputItemAddedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] output_index: int item: dict -class OutputItemDoneEvent(BaseModel): +class OutputItemDoneEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] output_index: int item: dict -class ContentPartAddedEvent(BaseModel): +class ContentPartAddedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] item_id: str output_index: int @@ -897,7 +909,7 @@ class ContentPartAddedEvent(BaseModel): part: dict -class ContentPartDoneEvent(BaseModel): +class ContentPartDoneEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] item_id: str output_index: int @@ -905,7 +917,7 @@ class ContentPartDoneEvent(BaseModel): part: dict -class OutputTextDeltaEvent(BaseModel): +class OutputTextDeltaEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] item_id: str output_index: int @@ -913,7 +925,7 @@ class OutputTextDeltaEvent(BaseModel): delta: str -class OutputTextAnnotationAddedEvent(BaseModel): +class OutputTextAnnotationAddedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED] item_id: str output_index: int @@ -922,7 +934,7 @@ class OutputTextAnnotationAddedEvent(BaseModel): annotation: dict -class OutputTextDoneEvent(BaseModel): +class OutputTextDoneEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] item_id: str output_index: int @@ -930,7 +942,7 @@ class OutputTextDoneEvent(BaseModel): text: str -class RefusalDeltaEvent(BaseModel): +class RefusalDeltaEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.REFUSAL_DELTA] item_id: str output_index: int @@ -938,7 +950,7 @@ class RefusalDeltaEvent(BaseModel): delta: str -class RefusalDoneEvent(BaseModel): +class RefusalDoneEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.REFUSAL_DONE] item_id: str output_index: int @@ -946,57 +958,57 @@ class RefusalDoneEvent(BaseModel): refusal: str -class FunctionCallArgumentsDeltaEvent(BaseModel): +class FunctionCallArgumentsDeltaEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] item_id: str output_index: int delta: str -class FunctionCallArgumentsDoneEvent(BaseModel): +class FunctionCallArgumentsDoneEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE] item_id: str output_index: int arguments: str -class FileSearchCallInProgressEvent(BaseModel): +class FileSearchCallInProgressEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS] output_index: int item_id: str -class FileSearchCallSearchingEvent(BaseModel): +class FileSearchCallSearchingEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING] output_index: int item_id: str -class FileSearchCallCompletedEvent(BaseModel): +class FileSearchCallCompletedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED] output_index: int item_id: str -class WebSearchCallInProgressEvent(BaseModel): +class WebSearchCallInProgressEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS] output_index: int item_id: str -class WebSearchCallSearchingEvent(BaseModel): +class WebSearchCallSearchingEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING] output_index: int item_id: str -class WebSearchCallCompletedEvent(BaseModel): +class WebSearchCallCompletedEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED] output_index: int item_id: str -class ErrorEvent(BaseModel): +class ErrorEvent(BaseResponseAPIStreamEvent): type: Literal[ResponsesAPIStreamEvents.ERROR] code: Optional[str] message: str From 122c11d346cdf8f9ce78cc223bc0f2872a1d2dd0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 07:14:36 -0700 Subject: [PATCH 34/61] revert to older logging implementation --- litellm/litellm_core_utils/litellm_logging.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 964bbfb70cd..cd5e9ef65d1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1002,8 +1002,10 @@ class Logging(LiteLLMLoggingBaseClass): ## if model in model cost map - log the response cost ## else set cost to None if ( - standard_logging_object is None and result is not None - ): # handle streaming separately + standard_logging_object is None + and result is not None + and self.stream is not True + ): if ( isinstance(result, ModelResponse) or isinstance(result, ModelResponseStream) @@ -1016,7 +1018,6 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(result, FineTuningJob) or isinstance(result, LiteLLMBatch) or isinstance(result, ResponsesAPIResponse) - or isinstance(result, ResponseCompletedEvent) ): ## HIDDEN PARAMS ## hidden_params = getattr(result, "_hidden_params", {}) From 46bc76d3e6a91c850890fe5a0e633e22d8f491c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 07:21:03 -0700 Subject: [PATCH 35/61] _get_assembled_streaming_response --- litellm/litellm_core_utils/litellm_logging.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cd5e9ef65d1..9acd70db6f9 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1117,7 +1117,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse] + Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent] ] = None if "complete_streaming_response" in self.model_call_details: return # break out of this. @@ -1639,7 +1639,7 @@ class Logging(LiteLLMLoggingBaseClass): if "async_complete_streaming_response" in self.model_call_details: return # break out of this. complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse] + Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent] ] = self._get_assembled_streaming_response( result=result, start_time=start_time, @@ -2349,16 +2349,24 @@ class Logging(LiteLLMLoggingBaseClass): def _get_assembled_streaming_response( self, - result: Union[ModelResponse, TextCompletionResponse, ModelResponseStream, Any], + result: Union[ + ModelResponse, + TextCompletionResponse, + ModelResponseStream, + ResponseCompletedEvent, + Any, + ], start_time: datetime.datetime, end_time: datetime.datetime, is_async: bool, streaming_chunks: List[Any], - ) -> Optional[Union[ModelResponse, TextCompletionResponse]]: + ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent]]: if isinstance(result, ModelResponse): return result elif isinstance(result, TextCompletionResponse): return result + elif isinstance(result, ResponseCompletedEvent): + return result elif isinstance(result, ModelResponseStream): complete_streaming_response: Optional[ Union[ModelResponse, TextCompletionResponse] From c2dbcb798f1eef29fe29049770d23c6d963d606d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 07:27:53 -0700 Subject: [PATCH 36/61] working streaming logging + cost tracking --- litellm/litellm_core_utils/litellm_logging.py | 8 +- .../test_openai_responses_api.py | 91 ++++++++++++------- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9acd70db6f9..18af6399180 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1117,7 +1117,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] ] = None if "complete_streaming_response" in self.model_call_details: return # break out of this. @@ -1639,7 +1639,7 @@ class Logging(LiteLLMLoggingBaseClass): if "async_complete_streaming_response" in self.model_call_details: return # break out of this. complete_streaming_response: Optional[ - Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent] + Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse] ] = self._get_assembled_streaming_response( result=result, start_time=start_time, @@ -2360,13 +2360,13 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: List[Any], - ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponseCompletedEvent]]: + ) -> Optional[Union[ModelResponse, TextCompletionResponse, ResponsesAPIResponse]]: if isinstance(result, ModelResponse): return result elif isinstance(result, TextCompletionResponse): return result elif isinstance(result, ResponseCompletedEvent): - return result + return result.response elif isinstance(result, ModelResponseStream): complete_streaming_response: Optional[ Union[ModelResponse, TextCompletionResponse] diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index efa6dcf4055..9745269befe 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -9,6 +9,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger import json from litellm.types.utils import StandardLoggingPayload +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse @pytest.mark.asyncio @@ -43,23 +44,66 @@ class TestCustomLogger(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): print("in async_log_success_event") + print("kwargs=", json.dumps(kwargs, indent=4, default=str)) self.standard_logging_object = kwargs["standard_logging_object"] pass +def validate_standard_logging_payload( + slp: StandardLoggingPayload, response: ResponsesAPIResponse, request_model: str +): + """ + Validate that a StandardLoggingPayload object matches the expected response + + Args: + slp (StandardLoggingPayload): The standard logging payload object to validate + response (dict): The litellm response to compare against + request_model (str): The model name that was requested + """ + # Validate payload exists + assert slp is not None, "Standard logging payload should not be None" + + # Validate token counts + print("response=", json.dumps(response, indent=4, default=str)) + assert ( + slp["prompt_tokens"] == response["usage"]["input_tokens"] + ), "Prompt tokens mismatch" + assert ( + slp["completion_tokens"] == response["usage"]["output_tokens"] + ), "Completion tokens mismatch" + assert ( + slp["total_tokens"] + == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] + ), "Total tokens mismatch" + + # Validate spend and response metadata + assert slp["response_cost"] > 0, "Response cost should be greater than 0" + assert slp["id"] == response["id"], "Response ID mismatch" + assert slp["model"] == request_model, "Model name mismatch" + + # Validate messages + assert slp["messages"] == [{"content": "hi", "role": "user"}], "Messages mismatch" + + # Validate complete response structure + validate_responses_match(slp["response"], response) + + @pytest.mark.asyncio async def test_basic_openai_responses_api_streaming_with_logging(): litellm._turn_on_debug() litellm.set_verbose = True test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] + request_model = "gpt-4o" response = await litellm.aresponses( - model="gpt-4o", + model=request_model, input="hi", stream=True, ) - + final_response: Optional[ResponseCompletedEvent] = None async for event in response: + if event.type == "response.completed": + final_response = event print("litellm response=", json.dumps(event, indent=4, default=str)) print("sleeping for 2 seconds...") @@ -69,6 +113,15 @@ async def test_basic_openai_responses_api_streaming_with_logging(): json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), ) + assert final_response is not None + assert test_custom_logger.standard_logging_object is not None + + validate_standard_logging_payload( + slp=test_custom_logger.standard_logging_object, + response=final_response.response, + request_model=request_model, + ) + def validate_responses_match(slp_response, litellm_response): """Validate that the standard logging payload OpenAI response matches the litellm response""" @@ -129,37 +182,9 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), ) + assert response is not None assert test_custom_logger.standard_logging_object is not None - # validate token counts match OpenAI response - assert ( - test_custom_logger.standard_logging_object["prompt_tokens"] - == response["usage"]["input_tokens"] - ) - assert ( - test_custom_logger.standard_logging_object["completion_tokens"] - == response["usage"]["output_tokens"] - ) - assert ( - test_custom_logger.standard_logging_object["total_tokens"] - == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] - ) - - # validate spend > 0 - assert test_custom_logger.standard_logging_object["response_cost"] > 0 - - # validate response id matches OpenAI - assert test_custom_logger.standard_logging_object["id"] == response["id"] - - # validate model matches - assert test_custom_logger.standard_logging_object["model"] == request_model - - # validate messages matches - assert test_custom_logger.standard_logging_object["messages"] == [ - {"content": "hi", "role": "user"} - ] - - # Add validation after existing assertions - validate_responses_match( - test_custom_logger.standard_logging_object["response"], response + validate_standard_logging_payload( + test_custom_logger.standard_logging_object, response, request_model ) From 3bf2fda128cbd22837143863321a5372fd551cb1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:17:27 -0700 Subject: [PATCH 37/61] add conftest --- tests/llm_responses_api_testing/conftest.py | 63 +++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/llm_responses_api_testing/conftest.py diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py new file mode 100644 index 00000000000..b3561d8a626 --- /dev/null +++ b/tests/llm_responses_api_testing/conftest.py @@ -0,0 +1,63 @@ +# conftest.py + +import importlib +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm + + +@pytest.fixture(scope="function", autouse=True) +def setup_and_teardown(): + """ + This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. + """ + curr_dir = os.getcwd() # Get the current working directory + sys.path.insert( + 0, os.path.abspath("../..") + ) # Adds the project directory to the system path + + import litellm + from litellm import Router + + importlib.reload(litellm) + + try: + if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): + import litellm.proxy.proxy_server + + importlib.reload(litellm.proxy.proxy_server) + except Exception as e: + print(f"Error reloading litellm.proxy.proxy_server: {e}") + + import asyncio + + loop = asyncio.get_event_loop_policy().new_event_loop() + asyncio.set_event_loop(loop) + print(litellm) + # from litellm import Router, completion, aembedding, acompletion, embedding + yield + + # Teardown code (executes after the yield point) + loop.close() # Close the loop created earlier + asyncio.set_event_loop(None) # Remove the reference to the loop + + +def pytest_collection_modifyitems(config, items): + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + custom_logger_tests = [ + item for item in items if "custom_logger" in item.parent.name + ] + other_tests = [item for item in items if "custom_logger" not in item.parent.name] + + # Sort tests based on their names + custom_logger_tests.sort(key=lambda x: x.name) + other_tests.sort(key=lambda x: x.name) + + # Reorder the items list + items[:] = custom_logger_tests + other_tests From e4cda0a1b7027a0f571ec515254f1f14383a399c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:18:25 -0700 Subject: [PATCH 38/61] add SyncResponsesAPIStreamingIterator --- litellm/responses/streaming_iterator.py | 186 +++++++++++++++++------- 1 file changed, 136 insertions(+), 50 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 7464d04607d..13112e3647f 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -17,12 +17,11 @@ from litellm.utils import CustomStreamWrapper COMPLETED_OPENAI_CHUNK_TYPE = "response.completed" -class ResponsesAPIStreamingIterator: +class BaseResponsesAPIStreamingIterator: """ - Async iterator for processing streaming responses from the Responses API. + Base class for streaming iterators that process responses from the Responses API. - This iterator handles the chunked streaming format returned by the Responses API - and yields properly formatted ResponsesAPIStreamingResponse objects. + This class contains shared logic for both synchronous and asynchronous iterators. """ def __init__( @@ -35,12 +34,76 @@ class ResponsesAPIStreamingIterator: self.response = response self.model = model self.logging_obj = logging_obj - self.stream_iterator = response.aiter_lines() self.finished = False self.responses_api_provider_config = responses_api_provider_config - self.completed_response: Optional[ResponsesAPIStreamingResponse] = None + self.completed_response = None self.start_time = datetime.now() + def _process_chunk(self, chunk): + """Process a single chunk of data from the stream""" + if not chunk: + return None + + # Handle SSE format (data: {...}) + chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) + if chunk is None: + return None + + # Handle "[DONE]" marker + if chunk == "[DONE]": + self.finished = True + return None + + try: + # Parse the JSON chunk + parsed_chunk = json.loads(chunk) + + # Format as ResponsesAPIStreamingResponse + if isinstance(parsed_chunk, dict): + openai_responses_api_chunk = ( + self.responses_api_provider_config.transform_streaming_response( + model=self.model, + parsed_chunk=parsed_chunk, + logging_obj=self.logging_obj, + ) + ) + # Store the completed response + if ( + openai_responses_api_chunk + and openai_responses_api_chunk.type + == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ): + self.completed_response = openai_responses_api_chunk + self._handle_completed_response() + + return openai_responses_api_chunk + + return None + except json.JSONDecodeError: + # If we can't parse the chunk, continue + return None + + def _handle_completed_response(self): + """Base implementation - should be overridden by subclasses""" + pass + + +class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): + """ + Async iterator for processing streaming responses from the Responses API. + """ + + def __init__( + self, + response: httpx.Response, + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + logging_obj: LiteLLMLoggingObj, + ): + super().__init__(response, model, responses_api_provider_config, logging_obj) + self.stream_iterator = response.aiter_lines() + self.completed_response: Optional[ResponsesAPIStreamingResponse] = None + def __aiter__(self): return self @@ -53,55 +116,78 @@ class ResponsesAPIStreamingIterator: self.finished = True raise StopAsyncIteration - if not chunk: - return await self.__anext__() + result = self._process_chunk(chunk) - # Handle SSE format (data: {...}) - chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) - if chunk is None: - return await self.__anext__() - - # Handle "[DONE]" marker - if chunk == "[DONE]": - self.finished = True + if self.finished: raise StopAsyncIteration - - try: - # Parse the JSON chunk - parsed_chunk = json.loads(chunk) - - # Format as ResponsesAPIStreamingResponse - if isinstance(parsed_chunk, dict): - openai_responses_api_chunk: ResponsesAPIStreamingResponse = ( - self.responses_api_provider_config.transform_streaming_response( - model=self.model, - parsed_chunk=parsed_chunk, - logging_obj=self.logging_obj, - ) - ) - # Store the completed response - if ( - openai_responses_api_chunk - and openai_responses_api_chunk.type - == ResponsesAPIStreamEvents.RESPONSE_COMPLETED - ): - self.completed_response = openai_responses_api_chunk - asyncio.create_task( - self.logging_obj.async_success_handler( - result=self.completed_response, - start_time=self.start_time, - end_time=datetime.now(), - cache_hit=None, - ) - ) - return openai_responses_api_chunk - - return await self.__anext__() - except json.JSONDecodeError: - # If we can't parse the chunk, continue to the next one + elif result is not None: + return result + else: return await self.__anext__() except httpx.HTTPError as e: # Handle HTTP errors self.finished = True raise e + + def _handle_completed_response(self): + """Handle logging for completed responses in async context""" + asyncio.create_task( + self.logging_obj.async_success_handler( + result=self.completed_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) + ) + + +class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): + """ + Synchronous iterator for processing streaming responses from the Responses API. + """ + + def __init__( + self, + response: httpx.Response, + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + logging_obj: LiteLLMLoggingObj, + ): + super().__init__(response, model, responses_api_provider_config, logging_obj) + self.stream_iterator = response.iter_lines() + + def __iter__(self): + return self + + def __next__(self): + try: + # Get the next chunk from the stream + try: + chunk = next(self.stream_iterator) + except StopIteration: + self.finished = True + raise StopIteration + + result = self._process_chunk(chunk) + + if self.finished: + raise StopIteration + elif result is not None: + return result + else: + return self.__next__() + + except httpx.HTTPError as e: + # Handle HTTP errors + self.finished = True + raise e + + def _handle_completed_response(self): + """Handle logging for completed responses in sync context""" + self.logging_obj.success_handler( + result=self.completed_response, + start_time=self.start_time, + end_time=datetime.now(), + cache_hit=None, + ) From 047879c004c7db24671cbabf3eba7ab949f8ea14 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:22:44 -0700 Subject: [PATCH 39/61] add aresponses --- litellm/llms/custom_httpx/llm_http_handler.py | 164 ++++++++++++++++-- litellm/responses/main.py | 128 ++++++++++---- .../test_openai_responses_api.py | 40 +++-- 3 files changed, 273 insertions(+), 59 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a303c4572cb..6f7671c3694 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,6 +1,6 @@ import io import json -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Union import httpx # type: ignore @@ -18,7 +18,11 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) -from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator +from litellm.responses.streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, @@ -961,32 +965,164 @@ class BaseLLMHTTPHandler: return returned_response return model_response - async def async_response_api_handler( + def response_api_handler( self, model: str, - custom_llm_provider: str, input: Union[str, ResponseInputParam], responses_api_provider_config: BaseResponsesAPIConfig, response_api_optional_request_params: Dict, - logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str, litellm_params: GenericLiteLLMParams, - client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + logging_obj: LiteLLMLoggingObj, extra_headers: Optional[Dict[str, Any]] = None, extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Union[ResponsesAPIResponse, ResponsesAPIStreamingIterator]: - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider) + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> Union[ + ResponsesAPIResponse, + BaseResponsesAPIStreamingIterator, + Coroutine[ + Any, Any, Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] + ], + ]: + """ + Handles responses API requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_response_api_handler( + model=model, + input=input, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_request_params=response_api_optional_request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client if isinstance(client, AsyncHTTPHandler) else None, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} ) else: - async_httpx_client = client + sync_httpx_client = client + headers = responses_api_provider_config.validate_environment( api_key=litellm_params.api_key, headers=response_api_optional_request_params.get("extra_headers", {}) or {}, model=model, ) + if extra_headers: + headers.update(extra_headers) + + api_base = responses_api_provider_config.get_complete_url( + api_base=litellm_params.api_base, + model=model, + ) + + data = responses_api_provider_config.transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + # Check if streaming is requested + stream = response_api_optional_request_params.get("stream", False) + + try: + if stream: + # For streaming, use stream=True in the request + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout + or response_api_optional_request_params.get("timeout"), + stream=True, + ) + + return SyncResponsesAPIStreamingIterator( + response=response, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + ) + else: + # For non-streaming requests + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout + or response_api_optional_request_params.get("timeout"), + ) + except Exception as e: + raise self._handle_error( + e=e, + provider_config=responses_api_provider_config, + ) + + return responses_api_provider_config.transform_response_api_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_response_api_handler( + self, + model: str, + input: Union[str, ResponseInputParam], + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: + """ + Async version of the responses API handler. + Uses async HTTP client to make requests. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = responses_api_provider_config.validate_environment( + api_key=litellm_params.api_key, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + ) + + if extra_headers: + headers.update(extra_headers) + api_base = responses_api_provider_config.get_complete_url( api_base=litellm_params.api_base, model=model, @@ -1021,7 +1157,8 @@ class BaseLLMHTTPHandler: url=api_base, headers=headers, data=json.dumps(data), - timeout=response_api_optional_request_params.get("timeout"), + timeout=timeout + or response_api_optional_request_params.get("timeout"), stream=True, ) @@ -1038,7 +1175,8 @@ class BaseLLMHTTPHandler: url=api_base, headers=headers, data=json.dumps(data), - timeout=response_api_optional_request_params.get("timeout"), + timeout=timeout + or response_api_optional_request_params.get("timeout"), ) except Exception as e: raise self._handle_error( diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 337e7fc3b07..62d3ddf215a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,3 +1,6 @@ +import asyncio +import contextvars +from functools import partial from typing import Any, Dict, Iterable, List, Literal, Optional, Union, get_type_hints import httpx @@ -23,7 +26,10 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client -from .streaming_iterator import ResponsesAPIStreamingIterator +from .streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, +) ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here @@ -75,9 +81,89 @@ async def aresponses( extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, **kwargs, -) -> Union[ResponsesAPIResponse, ResponsesAPIStreamingIterator]: +) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: + """ + Async: Handles responses API requests by reusing the synchronous function + """ + try: + loop = asyncio.get_event_loop() + kwargs["aresponses"] = True + + func = partial( + responses, + input, + model, + include, + instructions, + max_output_tokens, + metadata, + parallel_tool_calls, + previous_response_id, + reasoning, + store, + stream, + temperature, + text, + tool_choice, + tools, + top_p, + truncation, + user, + extra_headers, + extra_query, + extra_body, + timeout, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + return response + except Exception as e: + raise e + + +@client +def responses( + input: Union[str, ResponseInputParam], + model: str, + include: Optional[List[ResponseIncludable]] = None, + instructions: Optional[str] = None, + max_output_tokens: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + parallel_tool_calls: Optional[bool] = None, + previous_response_id: Optional[str] = None, + reasoning: Optional[Reasoning] = None, + store: Optional[bool] = None, + stream: Optional[bool] = None, + temperature: Optional[float] = None, + text: Optional[ResponseTextConfigParam] = None, + tool_choice: Optional[ToolChoice] = None, + tools: Optional[Iterable[ToolParam]] = None, + top_p: Optional[float] = None, + truncation: Optional[Literal["auto", "disabled"]] = None, + user: Optional[str] = None, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + **kwargs, +): + """ + Synchronous version of the Responses API. + Uses the synchronous HTTP handler to make requests. + """ litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("aresponses", False) is True # get llm provider logic litellm_params = GenericLiteLLMParams(**kwargs) @@ -132,7 +218,11 @@ async def aresponses( custom_llm_provider=custom_llm_provider, ) - response = await base_llm_http_handler.async_response_api_handler( + # Get an instance of BaseLLMHTTPHandler + base_llm_http_handler_instance = BaseLLMHTTPHandler() + + # Call the handler with _is_async flag instead of directly calling the async handler + response = base_llm_http_handler_instance.response_api_handler( model=model, input=input, responses_api_provider_config=responses_api_provider_config, @@ -143,34 +233,8 @@ async def aresponses( extra_headers=extra_headers, extra_body=extra_body, timeout=timeout, + _is_async=_is_async, + client=kwargs.get("client"), ) + return response - - -def responses( - input: Union[str, ResponseInputParam], - model: str, - include: Optional[List[ResponseIncludable]] = None, - instructions: Optional[str] = None, - max_output_tokens: Optional[int] = None, - metadata: Optional[Dict[str, Any]] = None, - parallel_tool_calls: Optional[bool] = None, - previous_response_id: Optional[str] = None, - reasoning: Optional[Reasoning] = None, - store: Optional[bool] = None, - stream: Optional[bool] = None, - temperature: Optional[float] = None, - text: Optional[ResponseTextConfigParam] = None, - tool_choice: Optional[ToolChoice] = None, - tools: Optional[Iterable[ToolParam]] = None, - top_p: Optional[float] = None, - truncation: Optional[Literal["auto", "disabled"]] = None, - user: Optional[str] = None, - # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. - # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Optional[Dict[str, Any]] = None, - extra_query: Optional[Dict[str, Any]] = None, - extra_body: Optional[Dict[str, Any]] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, -): - pass diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 9745269befe..15192c1b552 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -12,28 +12,40 @@ from litellm.types.utils import StandardLoggingPayload from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse +@pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio -async def test_basic_openai_responses_api(): +async def test_basic_openai_responses_api(sync_mode): litellm._turn_on_debug() - response = await litellm.aresponses( - model="gpt-4o", input="Tell me a three sentence bedtime story about a unicorn." - ) + + if sync_mode: + response = litellm.responses(model="gpt-4o", input="Basic ping") + else: + response = await litellm.aresponses(model="gpt-4o", input="Basic ping") + print("litellm response=", json.dumps(response, indent=4, default=str)) - # validate_responses_api_response() - +@pytest.mark.parametrize("sync_mode", [True]) @pytest.mark.asyncio -async def test_basic_openai_responses_api_streaming(): +async def test_basic_openai_responses_api_streaming(sync_mode): litellm._turn_on_debug() - response = await litellm.aresponses( - model="gpt-4o", - input="Tell me a three sentence bedtime story about a unicorn.", - stream=True, - ) - async for event in response: - print("litellm response=", json.dumps(event, indent=4, default=str)) + if sync_mode: + response = litellm.responses( + model="gpt-4o", + input="Basic ping", + stream=True, + ) + for event in response: + print("litellm response=", json.dumps(event, indent=4, default=str)) + else: + response = await litellm.aresponses( + model="gpt-4o", + input="Basic ping", + stream=True, + ) + async for event in response: + print("litellm response=", json.dumps(event, indent=4, default=str)) class TestCustomLogger(CustomLogger): From aa250088b23f4920110cf54aa02863048a79e0ef Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:31:34 -0700 Subject: [PATCH 40/61] re-use base_llm_http_handler --- litellm/responses/main.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 62d3ddf215a..50f040428a5 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -9,10 +9,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.responses.utils import ( - ResponsesAPIRequestParams, - get_optional_params_responses_api, -) +from litellm.responses.utils import get_optional_params_responses_api from litellm.types.llms.openai import ( Reasoning, ResponseIncludable, @@ -26,10 +23,7 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client -from .streaming_iterator import ( - BaseResponsesAPIStreamingIterator, - ResponsesAPIStreamingIterator, -) +from .streaming_iterator import BaseResponsesAPIStreamingIterator ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here @@ -218,11 +212,8 @@ def responses( custom_llm_provider=custom_llm_provider, ) - # Get an instance of BaseLLMHTTPHandler - base_llm_http_handler_instance = BaseLLMHTTPHandler() - # Call the handler with _is_async flag instead of directly calling the async handler - response = base_llm_http_handler_instance.response_api_handler( + response = base_llm_http_handler.response_api_handler( model=model, input=input, responses_api_provider_config=responses_api_provider_config, From 58acf23c3ee2656ed1a963497b7e3642c126fb4f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:33:28 -0700 Subject: [PATCH 41/61] STREAM_SSE_DONE_STRING --- litellm/constants.py | 1 + litellm/responses/streaming_iterator.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 0288c45e40b..b4551a78f5f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -18,6 +18,7 @@ SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = 1000 # Minimum number of requests REPEATED_STREAMING_CHUNK_LIMIT = 100 # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. #### Networking settings #### request_timeout: float = 6000 # time in seconds +STREAM_SSE_DONE_STRING: str = "[DONE]" LITELLM_CHAT_PROVIDERS = [ "openai", diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 13112e3647f..7325e4e6454 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -5,17 +5,15 @@ from typing import Any, AsyncIterator, Dict, Optional, Union import httpx +from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( - ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) from litellm.utils import CustomStreamWrapper -COMPLETED_OPENAI_CHUNK_TYPE = "response.completed" - class BaseResponsesAPIStreamingIterator: """ @@ -50,7 +48,7 @@ class BaseResponsesAPIStreamingIterator: return None # Handle "[DONE]" marker - if chunk == "[DONE]": + if chunk == STREAM_SSE_DONE_STRING: self.finished = True return None From ffa4978f8aec22ae9d25e2f5340e01533998b340 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:36:08 -0700 Subject: [PATCH 42/61] ResponsesAPIRequestUtils --- litellm/responses/main.py | 31 ++++--------- litellm/responses/utils.py | 95 +++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 50f040428a5..2450b53cac7 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -9,7 +9,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.responses.utils import get_optional_params_responses_api +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( Reasoning, ResponseIncludable, @@ -31,23 +31,6 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# -def get_requested_response_api_optional_param( - params: Dict[str, Any] -) -> ResponsesAPIOptionalRequestParams: - """ - Filter parameters to only include those defined in ResponsesAPIOptionalRequestParams. - - Args: - params: Dictionary of parameters to filter - - Returns: - ResponsesAPIOptionalRequestParams instance with only the valid parameters - """ - valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() - filtered_params = {k: v for k, v in params.items() if k in valid_keys} - return ResponsesAPIOptionalRequestParams(**filtered_params) - - @client async def aresponses( input: Union[str, ResponseInputParam], @@ -190,14 +173,16 @@ def responses( local_vars.update(kwargs) # Get ResponsesAPIOptionalRequestParams with only valid parameters response_api_optional_params: ResponsesAPIOptionalRequestParams = ( - get_requested_response_api_optional_param(local_vars) + ResponsesAPIRequestUtils.get_requested_response_api_optional_param(local_vars) ) # Get optional parameters for the responses API - responses_api_request_params: Dict = get_optional_params_responses_api( - model=model, - responses_api_provider_config=responses_api_provider_config, - response_api_optional_params=response_api_optional_params, + responses_api_request_params: Dict = ( + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=responses_api_provider_config, + response_api_optional_params=response_api_optional_params, + ) ) # Pre Call logging diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index c2775d9f579..3a148aed033 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,56 +1,79 @@ -import json -from typing import Any, Dict +from typing import Any, Dict, get_type_hints import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( ResponseAPIUsage, ResponsesAPIOptionalRequestParams, - ResponsesAPIRequestParams, ) from litellm.types.utils import Usage -def get_optional_params_responses_api( - model: str, - responses_api_provider_config: BaseResponsesAPIConfig, - response_api_optional_params: ResponsesAPIOptionalRequestParams, -) -> Dict: - """ - Get optional parameters for the responses API. +class ResponsesAPIRequestUtils: + """Helper utils for constructing ResponseAPI requests""" - Args: - params: Dictionary of all parameters - model: The model name - responses_api_provider_config: The provider configuration for responses API + @staticmethod + def get_optional_params_responses_api( + model: str, + responses_api_provider_config: BaseResponsesAPIConfig, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + ) -> Dict: + """ + Get optional parameters for the responses API. - Returns: - A dictionary of supported parameters for the responses API - """ - # Remove None values and internal parameters + Args: + params: Dictionary of all parameters + model: The model name + responses_api_provider_config: The provider configuration for responses API - # Get supported parameters for the model - supported_params = responses_api_provider_config.get_supported_openai_params(model) + Returns: + A dictionary of supported parameters for the responses API + """ + # Remove None values and internal parameters - # Check for unsupported parameters - unsupported_params = [ - param for param in response_api_optional_params if param not in supported_params - ] - - if unsupported_params: - raise litellm.UnsupportedParamsError( - model=model, - message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", + # Get supported parameters for the model + supported_params = responses_api_provider_config.get_supported_openai_params( + model ) - # Map parameters to provider-specific format - mapped_params = responses_api_provider_config.map_openai_params( - response_api_optional_params=response_api_optional_params, - model=model, - drop_params=litellm.drop_params, - ) + # Check for unsupported parameters + unsupported_params = [ + param + for param in response_api_optional_params + if param not in supported_params + ] - return mapped_params + if unsupported_params: + raise litellm.UnsupportedParamsError( + model=model, + message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}", + ) + + # Map parameters to provider-specific format + mapped_params = responses_api_provider_config.map_openai_params( + response_api_optional_params=response_api_optional_params, + model=model, + drop_params=litellm.drop_params, + ) + + return mapped_params + + @staticmethod + def get_requested_response_api_optional_param( + params: Dict[str, Any] + ) -> ResponsesAPIOptionalRequestParams: + """ + Filter parameters to only include those defined in ResponsesAPIOptionalRequestParams. + + Args: + params: Dictionary of parameters to filter + + Returns: + ResponsesAPIOptionalRequestParams instance with only the valid parameters + """ + valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() + filtered_params = {k: v for k, v in params.items() if k in valid_keys} + return ResponsesAPIOptionalRequestParams(**filtered_params) class ResponseAPILoggingUtils: From d6a49f6b6687f4ea2872d753e66a037f6da5b464 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:37:38 -0700 Subject: [PATCH 43/61] explictly pass params to partial func --- litellm/responses/main.py | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 2450b53cac7..ed5faa54d05 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -68,28 +68,28 @@ async def aresponses( func = partial( responses, - input, - model, - include, - instructions, - max_output_tokens, - metadata, - parallel_tool_calls, - previous_response_id, - reasoning, - store, - stream, - temperature, - text, - tool_choice, - tools, - top_p, - truncation, - user, - extra_headers, - extra_query, - extra_body, - timeout, + input=input, + model=model, + include=include, + instructions=instructions, + max_output_tokens=max_output_tokens, + metadata=metadata, + parallel_tool_calls=parallel_tool_calls, + previous_response_id=previous_response_id, + reasoning=reasoning, + store=store, + stream=stream, + temperature=temperature, + text=text, + tool_choice=tool_choice, + tools=tools, + top_p=top_p, + truncation=truncation, + user=user, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, **kwargs, ) From 35e9bba154a7f5cab96aa233a441f09fdb2c3878 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 09:42:48 -0700 Subject: [PATCH 44/61] _handle_logging_completed_response --- litellm/responses/streaming_iterator.py | 29 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 7325e4e6454..cc8711c3e10 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -6,7 +6,9 @@ from typing import Any, AsyncIterator, Dict, Optional, Union import httpx from litellm.constants import STREAM_SSE_DONE_STRING +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.types.llms.openai import ( ResponsesAPIStreamEvents, @@ -72,7 +74,7 @@ class BaseResponsesAPIStreamingIterator: == ResponsesAPIStreamEvents.RESPONSE_COMPLETED ): self.completed_response = openai_responses_api_chunk - self._handle_completed_response() + self._handle_logging_completed_response() return openai_responses_api_chunk @@ -81,7 +83,7 @@ class BaseResponsesAPIStreamingIterator: # If we can't parse the chunk, continue return None - def _handle_completed_response(self): + def _handle_logging_completed_response(self): """Base implementation - should be overridden by subclasses""" pass @@ -128,7 +130,7 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): self.finished = True raise e - def _handle_completed_response(self): + def _handle_logging_completed_response(self): """Handle logging for completed responses in async context""" asyncio.create_task( self.logging_obj.async_success_handler( @@ -139,6 +141,14 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): ) ) + executor.submit( + self.logging_obj.success_handler, + result=self.completed_response, + cache_hit=None, + start_time=self.start_time, + end_time=datetime.now(), + ) + class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): """ @@ -181,11 +191,20 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): self.finished = True raise e - def _handle_completed_response(self): + def _handle_logging_completed_response(self): """Handle logging for completed responses in sync context""" - self.logging_obj.success_handler( + run_async_function( + async_function=self.logging_obj.async_success_handler, result=self.completed_response, start_time=self.start_time, end_time=datetime.now(), cache_hit=None, ) + + executor.submit( + self.logging_obj.success_handler, + result=self.completed_response, + cache_hit=None, + start_time=self.start_time, + end_time=datetime.now(), + ) From d6351c34330d78a8aab664739eb8e08f1d883481 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 10:07:03 -0700 Subject: [PATCH 45/61] test_basic_openai_responses_api --- litellm/responses/main.py | 3 +- .../test_openai_responses_api.py | 76 ++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index ed5faa54d05..25e47dc5d9b 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,6 +6,7 @@ from typing import Any, Dict, Iterable, List, Literal, Optional, Union, get_type import httpx import litellm +from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -208,7 +209,7 @@ def responses( logging_obj=litellm_logging_obj, extra_headers=extra_headers, extra_body=extra_body, - timeout=timeout, + timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), ) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 15192c1b552..777a70dcafd 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -9,7 +9,78 @@ import litellm from litellm.integrations.custom_logger import CustomLogger import json from litellm.types.utils import StandardLoggingPayload -from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse +from litellm.types.llms.openai import ( + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponseTextConfig, +) + + +def validate_responses_api_response(response): + """ + Validate that a response from litellm.responses() or litellm.aresponses() + conforms to the expected ResponsesAPIResponse structure. + + Args: + response: The response object to validate + + Raises: + AssertionError: If the response doesn't match the expected structure + """ + # Validate response structure + print("response=", json.dumps(response, indent=4, default=str)) + assert isinstance( + response, ResponsesAPIResponse + ), "Response should be an instance of ResponsesAPIResponse" + + # Required fields + assert "id" in response and isinstance( + response["id"], str + ), "Response should have a string 'id' field" + assert "created_at" in response and isinstance( + response["created_at"], (int, float) + ), "Response should have a numeric 'created_at' field" + assert "output" in response and isinstance( + response["output"], list + ), "Response should have a list 'output' field" + assert "parallel_tool_calls" in response and isinstance( + response["parallel_tool_calls"], bool + ), "Response should have a boolean 'parallel_tool_calls' field" + + # Optional fields with their expected types + optional_fields = { + "error": (dict, type(None)), # error can be dict or None + "incomplete_details": (dict, type(None)), + "instructions": (str, type(None)), + "metadata": dict, + "model": str, + "object": str, + "temperature": (int, float), + "tool_choice": (dict, str), + "tools": list, + "top_p": (int, float), + "max_output_tokens": (int, type(None)), + "previous_response_id": (str, type(None)), + "reasoning": dict, + "status": str, + "text": ResponseTextConfig, + "truncation": str, + # "usage": dict, + "user": (str, type(None)), + } + + for field, expected_type in optional_fields.items(): + if field in response: + assert isinstance( + response[field], expected_type + ), f"Field '{field}' should be of type {expected_type}, but got {type(response[field])}" + + # Check if output has at least one item + assert ( + len(response["output"]) > 0 + ), "Response 'output' field should have at least one item" + + return True # Return True if validation passes @pytest.mark.parametrize("sync_mode", [True, False]) @@ -24,6 +95,9 @@ async def test_basic_openai_responses_api(sync_mode): print("litellm response=", json.dumps(response, indent=4, default=str)) + # Use the helper function to validate the response + validate_responses_api_response(response) + @pytest.mark.parametrize("sync_mode", [True]) @pytest.mark.asyncio From accdaa4a74a156ecc98586fd58f40c95413b14e6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:12:09 -0700 Subject: [PATCH 46/61] fix ResponseAPILoggingUtils --- litellm/cost_calculator.py | 17 ++++++++++++++--- .../test_openai_responses_api.py | 14 ++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index fb49c079812..209db5247dc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -45,7 +45,11 @@ from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_ai_image_cost_calculator, ) from litellm.responses.utils import ResponseAPILoggingUtils -from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse +from litellm.types.llms.openai import ( + HttpxBinaryResponseContent, + ResponseAPIUsage, + ResponsesAPIResponse, +) from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( CallTypesLiteral, @@ -465,6 +469,13 @@ def _get_usage_object( return usage_obj +def _is_known_usage_objects(usage_obj): + """Returns True if the usage obj is a known Usage type""" + return isinstance(usage_obj, litellm.Usage) or isinstance( + usage_obj, ResponseAPIUsage + ) + + def _infer_call_type( call_type: Optional[CallTypesLiteral], completion_response: Any ) -> Optional[CallTypesLiteral]: @@ -588,8 +599,8 @@ def completion_cost( # noqa: PLR0915 ) else: usage_obj = getattr(completion_response, "usage", {}) - if isinstance(usage_obj, BaseModel) and not isinstance( - usage_obj, litellm.Usage + if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects( + usage_obj=usage_obj ): setattr( completion_response, diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 777a70dcafd..a8324505059 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -13,6 +13,8 @@ from litellm.types.llms.openai import ( ResponseCompletedEvent, ResponsesAPIResponse, ResponseTextConfig, + ResponseAPIUsage, + IncompleteDetails, ) @@ -50,7 +52,7 @@ def validate_responses_api_response(response): # Optional fields with their expected types optional_fields = { "error": (dict, type(None)), # error can be dict or None - "incomplete_details": (dict, type(None)), + "incomplete_details": (IncompleteDetails, type(None)), "instructions": (str, type(None)), "metadata": dict, "model": str, @@ -65,7 +67,7 @@ def validate_responses_api_response(response): "status": str, "text": ResponseTextConfig, "truncation": str, - # "usage": dict, + "usage": ResponseAPIUsage, "user": (str, type(None)), } @@ -89,9 +91,13 @@ async def test_basic_openai_responses_api(sync_mode): litellm._turn_on_debug() if sync_mode: - response = litellm.responses(model="gpt-4o", input="Basic ping") + response = litellm.responses( + model="gpt-4o", input="Basic ping", max_output_tokens=20 + ) else: - response = await litellm.aresponses(model="gpt-4o", input="Basic ping") + response = await litellm.aresponses( + model="gpt-4o", input="Basic ping", max_output_tokens=20 + ) print("litellm response=", json.dumps(response, indent=4, default=str)) From daea59a7b48f631538884e2d035fef7ab2a3915d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:32:26 -0700 Subject: [PATCH 47/61] test openai responses streaming --- .../test_openai_responses_api.py | 233 +++++++++++++++++- 1 file changed, 228 insertions(+), 5 deletions(-) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index a8324505059..ca62c664d30 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -18,7 +18,7 @@ from litellm.types.llms.openai import ( ) -def validate_responses_api_response(response): +def validate_responses_api_response(response, final_chunk: bool = False): """ Validate that a response from litellm.responses() or litellm.aresponses() conforms to the expected ResponsesAPIResponse structure. @@ -70,6 +70,8 @@ def validate_responses_api_response(response): "usage": ResponseAPIUsage, "user": (str, type(None)), } + if final_chunk is False: + optional_fields["usage"] = type(None) for field, expected_type in optional_fields.items(): if field in response: @@ -78,9 +80,10 @@ def validate_responses_api_response(response): ), f"Field '{field}' should be of type {expected_type}, but got {type(response[field])}" # Check if output has at least one item - assert ( - len(response["output"]) > 0 - ), "Response 'output' field should have at least one item" + if final_chunk is True: + assert ( + len(response["output"]) > 0 + ), "Response 'output' field should have at least one item" return True # Return True if validation passes @@ -102,7 +105,7 @@ async def test_basic_openai_responses_api(sync_mode): print("litellm response=", json.dumps(response, indent=4, default=str)) # Use the helper function to validate the response - validate_responses_api_response(response) + validate_responses_api_response(response, final_chunk=True) @pytest.mark.parametrize("sync_mode", [True]) @@ -280,3 +283,223 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): validate_standard_logging_payload( test_custom_logger.standard_logging_object, response, request_model ) + + +def validate_stream_event(event): + """ + Validate that a streaming event from litellm.responses() or litellm.aresponses() + with stream=True conforms to the expected structure based on its event type. + + Args: + event: The streaming event object to validate + + Raises: + AssertionError: If the event doesn't match the expected structure for its type + """ + # Common validation for all event types + assert hasattr(event, "type"), "Event should have a 'type' attribute" + + # Type-specific validation + if event.type == "response.created" or event.type == "response.in_progress": + assert hasattr( + event, "response" + ), f"{event.type} event should have a 'response' attribute" + validate_responses_api_response(event.response, final_chunk=False) + + elif event.type == "response.completed": + assert hasattr( + event, "response" + ), "response.completed event should have a 'response' attribute" + validate_responses_api_response(event.response, final_chunk=True) + # Usage is guaranteed only on the completed event + assert ( + "usage" in event.response + ), "response.completed event should have usage information" + print("Usage in event.response=", event.response["usage"]) + assert isinstance(event.response["usage"], ResponseAPIUsage) + elif event.type == "response.failed" or event.type == "response.incomplete": + assert hasattr( + event, "response" + ), f"{event.type} event should have a 'response' attribute" + + elif ( + event.type == "response.output_item.added" + or event.type == "response.output_item.done" + ): + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "item" + ), f"{event.type} event should have an 'item' attribute" + + elif ( + event.type == "response.content_part.added" + or event.type == "response.content_part.done" + ): + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "part" + ), f"{event.type} event should have a 'part' attribute" + + elif event.type == "response.output_text.delta": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "delta" + ), f"{event.type} event should have a 'delta' attribute" + + elif event.type == "response.output_text.annotation.added": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "annotation_index" + ), f"{event.type} event should have an 'annotation_index' attribute" + assert hasattr( + event, "annotation" + ), f"{event.type} event should have an 'annotation' attribute" + + elif event.type == "response.output_text.done": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "text" + ), f"{event.type} event should have a 'text' attribute" + + elif event.type == "response.refusal.delta": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "delta" + ), f"{event.type} event should have a 'delta' attribute" + + elif event.type == "response.refusal.done": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "content_index" + ), f"{event.type} event should have a 'content_index' attribute" + assert hasattr( + event, "refusal" + ), f"{event.type} event should have a 'refusal' attribute" + + elif event.type == "response.function_call_arguments.delta": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "delta" + ), f"{event.type} event should have a 'delta' attribute" + + elif event.type == "response.function_call_arguments.done": + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "arguments" + ), f"{event.type} event should have an 'arguments' attribute" + + elif event.type in [ + "response.file_search_call.in_progress", + "response.file_search_call.searching", + "response.file_search_call.completed", + "response.web_search_call.in_progress", + "response.web_search_call.searching", + "response.web_search_call.completed", + ]: + assert hasattr( + event, "output_index" + ), f"{event.type} event should have an 'output_index' attribute" + assert hasattr( + event, "item_id" + ), f"{event.type} event should have an 'item_id' attribute" + + elif event.type == "error": + assert hasattr( + event, "message" + ), "Error event should have a 'message' attribute" + return True # Return True if validation passes + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_openai_responses_api_streaming_validation(sync_mode): + """Test that validates each streaming event from the responses API""" + litellm._turn_on_debug() + + event_types_seen = set() + + if sync_mode: + response = litellm.responses( + model="gpt-4o", + input="Tell me about artificial intelligence in 3 sentences.", + stream=True, + ) + for event in response: + print(f"Validating event type: {event.type}") + validate_stream_event(event) + event_types_seen.add(event.type) + else: + response = await litellm.aresponses( + model="gpt-4o", + input="Tell me about artificial intelligence in 3 sentences.", + stream=True, + ) + async for event in response: + print(f"Validating event type: {event.type}") + validate_stream_event(event) + event_types_seen.add(event.type) + + # At minimum, we should see these core event types + required_events = {"response.created", "response.completed"} + + missing_events = required_events - event_types_seen + assert not missing_events, f"Missing required event types: {missing_events}" + + print(f"Successfully validated all event types: {event_types_seen}") From e6b696370b4125569105e453f96d30c9857f0941 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:39:51 -0700 Subject: [PATCH 48/61] BaseLiteLLMOpenAIResponseObject --- litellm/types/llms/openai.py | 84 ++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 9e55ac30e99..0d28315e647 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -744,13 +744,24 @@ class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): model: str -class OutputTokensDetails(BaseModel): +class BaseLiteLLMOpenAIResponseObject(BaseModel): + def __getitem__(self, key): + return self.__dict__[key] + + def get(self, key, default=None): + return self.__dict__.get(key, default) + + def __contains__(self, key): + return key in self.__dict__ + + +class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): reasoning_tokens: int model_config = {"extra": "allow"} -class ResponseAPIUsage(BaseModel): +class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): input_tokens: int """The number of input tokens.""" @@ -766,7 +777,7 @@ class ResponseAPIUsage(BaseModel): model_config = {"extra": "allow"} -class ResponsesAPIResponse(BaseModel): +class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): id: str created_at: float error: Optional[dict] @@ -792,15 +803,6 @@ class ResponsesAPIResponse(BaseModel): # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) - def __getitem__(self, key): - return self.__dict__[key] - - def get(self, key, default=None): - return self.__dict__.get(key, default) - - def __contains__(self, key): - return key in self.__dict__ - class ResponsesAPIStreamEvents(str, Enum): """ @@ -851,57 +853,45 @@ class ResponsesAPIStreamEvents(str, Enum): ERROR = "error" -# Base streaming response types -class BaseResponseAPIStreamEvent(BaseModel): - def __getitem__(self, key): - return self.__dict__[key] - - def get(self, key, default=None): - return self.__dict__.get(key, default) - - def __contains__(self, key): - return key in self.__dict__ - - -class ResponseCreatedEvent(BaseResponseAPIStreamEvent): +class ResponseCreatedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_CREATED] response: ResponsesAPIResponse -class ResponseInProgressEvent(BaseResponseAPIStreamEvent): +class ResponseInProgressEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS] response: ResponsesAPIResponse -class ResponseCompletedEvent(BaseResponseAPIStreamEvent): +class ResponseCompletedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_COMPLETED] response: ResponsesAPIResponse _hidden_params: dict = PrivateAttr(default_factory=dict) -class ResponseFailedEvent(BaseResponseAPIStreamEvent): +class ResponseFailedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_FAILED] response: ResponsesAPIResponse -class ResponseIncompleteEvent(BaseResponseAPIStreamEvent): +class ResponseIncompleteEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] response: ResponsesAPIResponse -class OutputItemAddedEvent(BaseResponseAPIStreamEvent): +class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] output_index: int item: dict -class OutputItemDoneEvent(BaseResponseAPIStreamEvent): +class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] output_index: int item: dict -class ContentPartAddedEvent(BaseResponseAPIStreamEvent): +class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] item_id: str output_index: int @@ -909,7 +899,7 @@ class ContentPartAddedEvent(BaseResponseAPIStreamEvent): part: dict -class ContentPartDoneEvent(BaseResponseAPIStreamEvent): +class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] item_id: str output_index: int @@ -917,7 +907,7 @@ class ContentPartDoneEvent(BaseResponseAPIStreamEvent): part: dict -class OutputTextDeltaEvent(BaseResponseAPIStreamEvent): +class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA] item_id: str output_index: int @@ -925,7 +915,7 @@ class OutputTextDeltaEvent(BaseResponseAPIStreamEvent): delta: str -class OutputTextAnnotationAddedEvent(BaseResponseAPIStreamEvent): +class OutputTextAnnotationAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED] item_id: str output_index: int @@ -934,7 +924,7 @@ class OutputTextAnnotationAddedEvent(BaseResponseAPIStreamEvent): annotation: dict -class OutputTextDoneEvent(BaseResponseAPIStreamEvent): +class OutputTextDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE] item_id: str output_index: int @@ -942,7 +932,7 @@ class OutputTextDoneEvent(BaseResponseAPIStreamEvent): text: str -class RefusalDeltaEvent(BaseResponseAPIStreamEvent): +class RefusalDeltaEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.REFUSAL_DELTA] item_id: str output_index: int @@ -950,7 +940,7 @@ class RefusalDeltaEvent(BaseResponseAPIStreamEvent): delta: str -class RefusalDoneEvent(BaseResponseAPIStreamEvent): +class RefusalDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.REFUSAL_DONE] item_id: str output_index: int @@ -958,57 +948,57 @@ class RefusalDoneEvent(BaseResponseAPIStreamEvent): refusal: str -class FunctionCallArgumentsDeltaEvent(BaseResponseAPIStreamEvent): +class FunctionCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] item_id: str output_index: int delta: str -class FunctionCallArgumentsDoneEvent(BaseResponseAPIStreamEvent): +class FunctionCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE] item_id: str output_index: int arguments: str -class FileSearchCallInProgressEvent(BaseResponseAPIStreamEvent): +class FileSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS] output_index: int item_id: str -class FileSearchCallSearchingEvent(BaseResponseAPIStreamEvent): +class FileSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING] output_index: int item_id: str -class FileSearchCallCompletedEvent(BaseResponseAPIStreamEvent): +class FileSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED] output_index: int item_id: str -class WebSearchCallInProgressEvent(BaseResponseAPIStreamEvent): +class WebSearchCallInProgressEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS] output_index: int item_id: str -class WebSearchCallSearchingEvent(BaseResponseAPIStreamEvent): +class WebSearchCallSearchingEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING] output_index: int item_id: str -class WebSearchCallCompletedEvent(BaseResponseAPIStreamEvent): +class WebSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED] output_index: int item_id: str -class ErrorEvent(BaseResponseAPIStreamEvent): +class ErrorEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.ERROR] code: Optional[str] message: str From 1b50e6d65fb7aa0016c499786667aa531b176db4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:44:16 -0700 Subject: [PATCH 49/61] openai 1.66.1 --- .circleci/config.yml | 10 +++++----- requirements.txt | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 68f9ee14cda..4ddfe3a0ab6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1276,7 +1276,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0 " + pip install "openai==1.54.0" - run: name: Install Grype command: | @@ -1412,7 +1412,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0 " + pip install "openai==1.66.1" # Run pytest and generate JUnit XML report - run: name: Build Docker image @@ -1534,7 +1534,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0 " + pip install "openai==1.66.1" - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -1963,7 +1963,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "google-cloud-aiplatform==1.43.0" pip install aiohttp - pip install "openai==1.54.0 " + pip install "openai==1.66.1" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip pip install "pydantic==2.7.1" @@ -2239,7 +2239,7 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install "openai==1.54.0 " + pip install "openai==1.66.1" python -m pip install --upgrade pip pip install "pydantic==2.7.1" pip install "pytest==7.3.1" diff --git a/requirements.txt b/requirements.txt index 3d695d17662..dcdddff1179 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # anyio==4.4.0 # openai + http req. httpx==0.27.0 # Pin Httpx dependency -openai==1.61.0 # openai req. +openai==1.66.1 # openai req. fastapi==0.115.5 # server dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep From 181072e15d11a2f3ddc0150c3b5200b8379972d9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:47:10 -0700 Subject: [PATCH 50/61] fix code quality checks --- litellm/llms/base_llm/responses/transformation.py | 1 - litellm/llms/custom_httpx/llm_http_handler.py | 7 +------ litellm/responses/main.py | 2 +- litellm/responses/streaming_iterator.py | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 6e27fd7a542..c41d63842b1 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -12,7 +12,6 @@ from litellm.types.llms.openai import ( ResponsesAPIStreamingResponse, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ModelInfo if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6f7671c3694..6f8fd347c14 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -23,12 +23,7 @@ from litellm.responses.streaming_iterator import ( ResponsesAPIStreamingIterator, SyncResponsesAPIStreamingIterator, ) -from litellm.types.llms.openai import ( - ResponseInputParam, - ResponsesAPIOptionalRequestParams, - ResponsesAPIRequestParams, - ResponsesAPIResponse, -) +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse from litellm.types.rerank import OptionalRerankParams, RerankResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 25e47dc5d9b..ce70292e962 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars from functools import partial -from typing import Any, Dict, Iterable, List, Literal, Optional, Union, get_type_hints +from typing import Any, Dict, Iterable, List, Literal, Optional, Union import httpx diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index cc8711c3e10..e0d9e292c65 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1,7 +1,7 @@ import asyncio import json from datetime import datetime -from typing import Any, AsyncIterator, Dict, Optional, Union +from typing import Optional import httpx From 595af6898fab3d606aadccb0cc81d274fd3c1a9d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:51:22 -0700 Subject: [PATCH 51/61] llm_responses_api_testing --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ddfe3a0ab6..6dd4f14f886 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -718,8 +718,8 @@ jobs: - persist_to_workspace: root: . paths: - - llm_responses_api_testing.xml - - llm_responses_api_testing + - llm_responses_api_coverage.xml + - llm_responses_api_coverage litellm_mapped_tests: docker: - image: cimg/python:3.11 From 9ea0c89a2da71d2cb1343ea80e4a8d1ea42e1af9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:52:03 -0700 Subject: [PATCH 52/61] bump to openai==1.66.1 --- .circleci/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 12e83a40f29..e63fb9dd9a9 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -1,5 +1,5 @@ # used by CI/CD testing -openai==1.54.0 +openai==1.66.1 python-dotenv tiktoken importlib_metadata From 1f7c21fd1b6ce11fe79122a432a92c0e9b783a85 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:55:17 -0700 Subject: [PATCH 53/61] remove infinit loop for streaming --- litellm/responses/streaming_iterator.py | 52 ++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index e0d9e292c65..88d963c94ed 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -109,21 +109,21 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): async def __anext__(self) -> ResponsesAPIStreamingResponse: try: - # Get the next chunk from the stream - try: - chunk = await self.stream_iterator.__anext__() - except StopAsyncIteration: - self.finished = True - raise StopAsyncIteration + while True: + # Get the next chunk from the stream + try: + chunk = await self.stream_iterator.__anext__() + except StopAsyncIteration: + self.finished = True + raise StopAsyncIteration - result = self._process_chunk(chunk) + result = self._process_chunk(chunk) - if self.finished: - raise StopAsyncIteration - elif result is not None: - return result - else: - return await self.__anext__() + if self.finished: + raise StopAsyncIteration + elif result is not None: + return result + # If result is None, continue the loop to get the next chunk except httpx.HTTPError as e: # Handle HTTP errors @@ -170,21 +170,21 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): def __next__(self): try: - # Get the next chunk from the stream - try: - chunk = next(self.stream_iterator) - except StopIteration: - self.finished = True - raise StopIteration + while True: + # Get the next chunk from the stream + try: + chunk = next(self.stream_iterator) + except StopIteration: + self.finished = True + raise StopIteration - result = self._process_chunk(chunk) + result = self._process_chunk(chunk) - if self.finished: - raise StopIteration - elif result is not None: - return result - else: - return self.__next__() + if self.finished: + raise StopIteration + elif result is not None: + return result + # If result is None, continue the loop to get the next chunk except httpx.HTTPError as e: # Handle HTTP errors From 6702e492f5a5df642c73df6fca7cfbe5820d1572 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:56:34 -0700 Subject: [PATCH 54/61] pip install openai==1.66.1 --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6dd4f14f886..e1bbe07889f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -168,7 +168,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -267,7 +267,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -511,7 +511,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 + pip install openai==1.66.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" From c6a9e8cafe2c8d542747405bd0b2d9de25680cf1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 11:58:56 -0700 Subject: [PATCH 55/61] typing_extensions Annotated --- litellm/types/llms/openai.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0d28315e647..58cb2ab479d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,17 +1,6 @@ from enum import Enum from os import PathLike -from typing import ( - IO, - Annotated, - Any, - Iterable, - List, - Literal, - Mapping, - Optional, - Tuple, - Union, -) +from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union import httpx from openai._legacy_response import ( @@ -62,7 +51,7 @@ from openai.types.responses.response_create_params import ( ToolParam, ) from pydantic import BaseModel, Discriminator, Field, PrivateAttr -from typing_extensions import Dict, Required, TypedDict, override +from typing_extensions import Annotated, Dict, Required, TypedDict, override FileContent = Union[IO[bytes], bytes, PathLike] From de473bee4bdbf4104dff11b55aa19dd279dd5dce Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 12:13:19 -0700 Subject: [PATCH 56/61] fix mypy linting errors --- litellm/responses/streaming_iterator.py | 3 +-- litellm/responses/utils.py | 4 ++-- litellm/types/llms/openai.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 88d963c94ed..c016e71e7e2 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -36,7 +36,7 @@ class BaseResponsesAPIStreamingIterator: self.logging_obj = logging_obj self.finished = False self.responses_api_provider_config = responses_api_provider_config - self.completed_response = None + self.completed_response: Optional[ResponsesAPIStreamingResponse] = None self.start_time = datetime.now() def _process_chunk(self, chunk): @@ -102,7 +102,6 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): ): super().__init__(response, model, responses_api_provider_config, logging_obj) self.stream_iterator = response.aiter_lines() - self.completed_response: Optional[ResponsesAPIStreamingResponse] = None def __aiter__(self): return self diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 3a148aed033..49d850ec6aa 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, get_type_hints +from typing import Any, Dict, cast, get_type_hints import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -73,7 +73,7 @@ class ResponsesAPIRequestUtils: """ valid_keys = get_type_hints(ResponsesAPIOptionalRequestParams).keys() filtered_params = {k: v for k, v in params.items() if k in valid_keys} - return ResponsesAPIOptionalRequestParams(**filtered_params) + return cast(ResponsesAPIOptionalRequestParams, filtered_params) class ResponseAPILoggingUtils: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 58cb2ab479d..5b811830e70 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -35,7 +35,6 @@ from openai.types.embedding import Embedding as OpenAIEmbedding from openai.types.fine_tuning.fine_tuning_job import FineTuningJob from openai.types.responses.response import ( IncompleteDetails, - Reasoning, Response, ResponseOutputItem, ResponseTextConfig, From 39d391d8e7d0b6e4579df2710c03444041bbef5c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 12:29:13 -0700 Subject: [PATCH 57/61] Optional[Dict] --- litellm/llms/azure/assistants.py | 18 +++++++++--------- litellm/llms/openai/openai.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/litellm/llms/azure/assistants.py b/litellm/llms/azure/assistants.py index 2f67b5506f0..b446d887c40 100644 --- a/litellm/llms/azure/assistants.py +++ b/litellm/llms/azure/assistants.py @@ -1,4 +1,4 @@ -from typing import Coroutine, Iterable, Literal, Optional, Union +from typing import Any, Coroutine, Dict, Iterable, Literal, Optional, Union import httpx from openai import AsyncAzureOpenAI, AzureOpenAI @@ -618,7 +618,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -659,12 +659,12 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -684,12 +684,12 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AssistantStreamManager[AssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -711,7 +711,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -733,7 +733,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -756,7 +756,7 @@ class AzureAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 7935c46293c..880a043d08a 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -2650,7 +2650,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -2689,12 +2689,12 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -2714,12 +2714,12 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], tools: Optional[Iterable[AssistantToolParam]], event_handler: Optional[AssistantEventHandler], ) -> AssistantStreamManager[AssistantEventHandler]: - data = { + data: Dict[str, Any] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -2741,7 +2741,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -2763,7 +2763,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], @@ -2786,7 +2786,7 @@ class OpenAIAssistantsAPI(BaseLLM): assistant_id: str, additional_instructions: Optional[str], instructions: Optional[str], - metadata: Optional[object], + metadata: Optional[Dict], model: Optional[str], stream: Optional[bool], tools: Optional[Iterable[AssistantToolParam]], From f6f5420f0a2347728703943ba436672db7c6701a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 12:42:20 -0700 Subject: [PATCH 58/61] TestResponsesAPIRequestUtils --- .../litellm/responses/test_responses_utils.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/litellm/responses/test_responses_utils.py diff --git a/tests/litellm/responses/test_responses_utils.py b/tests/litellm/responses/test_responses_utils.py new file mode 100644 index 00000000000..ab770552db4 --- /dev/null +++ b/tests/litellm/responses/test_responses_utils.py @@ -0,0 +1,84 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +import litellm +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + +class TestResponsesAPIRequestUtils: + def test_get_optional_params_responses_api(self): + """Test that optional parameters are correctly processed for responses API""" + # Setup + model = "gpt-4o" + config = OpenAIResponsesAPIConfig() + optional_params = ResponsesAPIOptionalRequestParams( + {"temperature": 0.7, "max_output_tokens": 100} + ) + + # Execute + result = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=config, + response_api_optional_params=optional_params, + ) + + # Assert + assert result == optional_params + assert "temperature" in result + assert result["temperature"] == 0.7 + assert "max_output_tokens" in result + assert result["max_output_tokens"] == 100 + + def test_get_optional_params_responses_api_unsupported_param(self): + """Test that unsupported parameters raise an error""" + # Setup + model = "gpt-4o" + config = OpenAIResponsesAPIConfig() + optional_params = ResponsesAPIOptionalRequestParams( + {"temperature": 0.7, "unsupported_param": "value"} + ) + + # Execute and Assert + with pytest.raises(litellm.UnsupportedParamsError) as excinfo: + ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=config, + response_api_optional_params=optional_params, + ) + + assert "unsupported_param" in str(excinfo.value) + assert model in str(excinfo.value) + + def test_get_requested_response_api_optional_param(self): + """Test filtering parameters to only include those in ResponsesAPIOptionalRequestParams""" + # Setup + params = { + "temperature": 0.7, + "max_output_tokens": 100, + "invalid_param": "value", + "model": "gpt-4o", # This is not in ResponsesAPIOptionalRequestParams + } + + # Execute + result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + params + ) + + # Assert + assert "temperature" in result + assert "max_output_tokens" in result + assert "invalid_param" not in result + assert "model" not in result + assert result["temperature"] == 0.7 + assert result["max_output_tokens"] == 100 From f88380cfdf78fe26d8045c6af43d07a011f1a9e1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 12:45:03 -0700 Subject: [PATCH 59/61] TestResponseAPILoggingUtils --- .../litellm/responses/test_responses_utils.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/litellm/responses/test_responses_utils.py b/tests/litellm/responses/test_responses_utils.py index ab770552db4..3567f609e73 100644 --- a/tests/litellm/responses/test_responses_utils.py +++ b/tests/litellm/responses/test_responses_utils.py @@ -14,6 +14,7 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfi from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.utils import Usage class TestResponsesAPIRequestUtils: @@ -82,3 +83,68 @@ class TestResponsesAPIRequestUtils: assert "model" not in result assert result["temperature"] == 0.7 assert result["max_output_tokens"] == 100 + + +class TestResponseAPILoggingUtils: + def test_is_response_api_usage_true(self): + """Test identification of Response API usage format""" + # Setup + usage = {"input_tokens": 10, "output_tokens": 20} + + # Execute + result = ResponseAPILoggingUtils._is_response_api_usage(usage) + + # Assert + assert result is True + + def test_is_response_api_usage_false(self): + """Test identification of non-Response API usage format""" + # Setup + usage = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30} + + # Execute + result = ResponseAPILoggingUtils._is_response_api_usage(usage) + + # Assert + assert result is False + + def test_transform_response_api_usage_to_chat_usage(self): + """Test transformation from Response API usage to Chat usage format""" + # Setup + usage = { + "input_tokens": 10, + "output_tokens": 20, + "total_tokens": 30, + "output_tokens_details": {"reasoning_tokens": 5}, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert + assert isinstance(result, Usage) + assert result.prompt_tokens == 10 + assert result.completion_tokens == 20 + assert result.total_tokens == 30 + + def test_transform_response_api_usage_with_none_values(self): + """Test transformation handles None values properly""" + # Setup + usage = { + "input_tokens": 0, # Changed from None to 0 + "output_tokens": 20, + "total_tokens": 20, + "output_tokens_details": {"reasoning_tokens": 5}, + } + + # Execute + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + # Assert + assert result.prompt_tokens == 0 + assert result.completion_tokens == 20 + assert result.total_tokens == 20 From 2460f3cbaba7fe6366bbb2dc406291fa906b6dda Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 12:57:40 -0700 Subject: [PATCH 60/61] test_validate_environment --- .../llms/openai/responses/transformation.py | 3 + .../test_openai_responses_transformation.py | 239 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 tests/litellm/llms/openai/responses/test_openai_responses_transformation.py diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 84641829db5..ce4052dc197 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -55,6 +55,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model: str, drop_params: bool, ) -> Dict: + """No mapping applied since inputs are in OpenAI spec already""" return dict(response_api_optional_params) def transform_responses_api_request( @@ -65,6 +66,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> ResponsesAPIRequestParams: + """No transform applied since inputs are in OpenAI spec already""" return ResponsesAPIRequestParams( model=model, input=input, **response_api_optional_request_params ) @@ -75,6 +77,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> ResponsesAPIResponse: + """No transform applied since outputs are in OpenAI spec already""" try: raw_response_json = raw_response.json() except Exception: diff --git a/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py new file mode 100644 index 00000000000..b4a6cd974ef --- /dev/null +++ b/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -0,0 +1,239 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) + + +class TestOpenAIResponsesAPIConfig: + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + self.model = "gpt-4o" + self.logging_obj = MagicMock() + + def test_map_openai_params(self): + """Test that parameters are correctly mapped""" + test_params = {"input": "Hello world", "temperature": 0.7, "stream": True} + + result = self.config.map_openai_params( + response_api_optional_params=test_params, + model=self.model, + drop_params=False, + ) + + # The function should return the params unchanged + assert result == test_params + + def validate_responses_api_request_params(self, params, expected_fields): + """ + Validate that the params dict has the expected structure of ResponsesAPIRequestParams + + Args: + params: The dict to validate + expected_fields: Dict of field names and their expected values + """ + # Check that it's a dict + assert isinstance(params, dict), "Result should be a dict" + + # Check expected fields have correct values + for field, value in expected_fields.items(): + assert field in params, f"Missing expected field: {field}" + assert ( + params[field] == value + ), f"Field {field} has value {params[field]}, expected {value}" + + def test_transform_responses_api_request(self): + """Test request transformation""" + input_text = "What is the capital of France?" + optional_params = {"temperature": 0.7, "stream": True} + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_text, + response_api_optional_request_params=optional_params, + litellm_params={}, + headers={}, + ) + + # Validate the result has the expected structure and values + expected_fields = { + "model": self.model, + "input": input_text, + "temperature": 0.7, + "stream": True, + } + + self.validate_responses_api_request_params(result, expected_fields) + + def test_transform_streaming_response(self): + """Test streaming response transformation""" + # Test with a text delta event + chunk = { + "type": "response.output_text.delta", + "item_id": "item_123", + "output_index": 0, + "content_index": 0, + "delta": "Hello", + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputTextDeltaEvent) + assert result.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + assert result.delta == "Hello" + assert result.item_id == "item_123" + + # Test with a completed event - providing all required fields + completed_chunk = { + "type": "response.completed", + "response": { + "id": "resp_123", + "created_at": 1234567890, + "model": "gpt-4o", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": 0.7, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "status": "completed", + "text": None, + "truncation": "auto", + "usage": None, + "user": None, + }, + } + + # Mock the get_event_model_class to avoid validation issues in tests + with patch.object( + OpenAIResponsesAPIConfig, "get_event_model_class" + ) as mock_get_class: + mock_get_class.return_value = ResponseCompletedEvent + + result = self.config.transform_streaming_response( + model=self.model, + parsed_chunk=completed_chunk, + logging_obj=self.logging_obj, + ) + + assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + assert result.response.id == "resp_123" + + def test_validate_environment(self): + """Test that validate_environment correctly sets the Authorization header""" + # Test with provided API key + headers = {} + api_key = "test_api_key" + + result = self.config.validate_environment( + headers=headers, model=self.model, api_key=api_key + ) + + assert "Authorization" in result + assert result["Authorization"] == f"Bearer {api_key}" + + # Test with empty headers + headers = {} + + with patch("litellm.api_key", "litellm_api_key"): + result = self.config.validate_environment(headers=headers, model=self.model) + + assert "Authorization" in result + assert result["Authorization"] == "Bearer litellm_api_key" + + # Test with existing headers + headers = {"Content-Type": "application/json"} + + with patch("litellm.openai_key", "openai_key"): + with patch("litellm.api_key", None): + result = self.config.validate_environment( + headers=headers, model=self.model + ) + + assert "Authorization" in result + assert result["Authorization"] == "Bearer openai_key" + assert "Content-Type" in result + assert result["Content-Type"] == "application/json" + + # Test with environment variable + headers = {} + + with patch("litellm.api_key", None): + with patch("litellm.openai_key", None): + with patch( + "litellm.llms.openai.responses.transformation.get_secret_str", + return_value="env_api_key", + ): + result = self.config.validate_environment( + headers=headers, model=self.model + ) + + assert "Authorization" in result + assert result["Authorization"] == "Bearer env_api_key" + + def test_get_complete_url(self): + """Test that get_complete_url returns the correct URL""" + # Test with provided API base + api_base = "https://custom-openai.example.com/v1" + + result = self.config.get_complete_url(api_base=api_base, model=self.model) + + assert result == "https://custom-openai.example.com/v1/responses" + + # Test with litellm.api_base + with patch("litellm.api_base", "https://litellm-api-base.example.com/v1"): + result = self.config.get_complete_url(api_base=None, model=self.model) + + assert result == "https://litellm-api-base.example.com/v1/responses" + + # Test with environment variable + with patch("litellm.api_base", None): + with patch( + "litellm.llms.openai.responses.transformation.get_secret_str", + return_value="https://env-api-base.example.com/v1", + ): + result = self.config.get_complete_url(api_base=None, model=self.model) + + assert result == "https://env-api-base.example.com/v1/responses" + + # Test with default API base + with patch("litellm.api_base", None): + with patch( + "litellm.llms.openai.responses.transformation.get_secret_str", + return_value=None, + ): + result = self.config.get_complete_url(api_base=None, model=self.model) + + assert result == "https://api.openai.com/v1/responses" + + # Test with trailing slash in API base + api_base = "https://custom-openai.example.com/v1/" + + result = self.config.get_complete_url(api_base=api_base, model=self.model) + + assert result == "https://custom-openai.example.com/v1/responses" From aa72f95a06a3353d3a997153975bb2d59fa2fdde Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 12 Mar 2025 13:00:59 -0700 Subject: [PATCH 61/61] "openai==1.66.1" in testing --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1bbe07889f..71580874453 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1276,7 +1276,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.54.0" + pip install "openai==1.66.1" - run: name: Install Grype command: |