diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 9ebb9bab018..19d4619092a 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -424,6 +424,7 @@ jobs: tests/unit/images tests/unit/interactions tests/unit/messages + tests/unit/native tests/unit/ocr tests/unit/passthrough tests/unit/rag @@ -715,6 +716,7 @@ jobs: test-path: |- tests/unit/responses/litellm_completion_transformation tests/unit/responses/test_additional_tools.py + tests/unit/responses/test_bridge_executor.py tests/unit/responses/test_custom_tool_call.py tests/unit/responses/test_metadata_codex_callback.py tests/unit/responses/test_no_duplicate_spend_logs.py diff --git a/Makefile b/Makefile index 48df3c8e215..d571ca6fc2a 100644 --- a/Makefile +++ b/Makefile @@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) # Linting targets lint-ruff: $(LINT_DEP_INSTALL) $(UV_RUN) python scripts/check_mcp_operation_boundary.py + $(UV_RUN) python scripts/check_native_imports.py cd litellm && $(UV_RUN) ruff check . && cd .. $(UV_RUN) ruff check --config ruff-tests.toml tests diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 76d17ea9693..9e683fe4de3 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -11,8 +11,10 @@ from typing import ( cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict ) +from pydantic import TypeAdapter, ValidationError from typing_extensions import TypedDict +import litellm from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: @@ -36,6 +38,60 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: return f"{custom_llm_provider}/{model}" +_TOOLS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_BRIDGED_REQUEST: Final = TypeAdapter(dict[str, object]) +_TOOLS_ONLY_RESPONSES_SERVES: Final = frozenset({"mcp", "file_search"}) + + +def _needs_responses_surface(request_data: Mapping[str, object]) -> bool: + """`responses()` also checks the cache and dispatches MCP and emulated file_search tools, which the executor skips.""" + if litellm.cache is not None or request_data.get("use_chat_completions_api"): + return True + model: Final = request_data.get("model") + if not isinstance(model, str) or model.startswith("openai/chat_completions/"): + return True + try: + tools: Final = _TOOLS.validate_python(request_data.get("tools") or ()) + except ValidationError: + return True + return any(tool.get("type") in _TOOLS_ONLY_RESPONSES_SERVES for tool in tools) + + +def _executor_logging_obj( + request_data: Mapping[str, object], logging_obj: "LiteLLMLoggingObj" +) -> "LiteLLMLoggingObj | None": + from litellm.responses.bridge_executor import has_native_responses_config, native_logging_obj + + model: Final = request_data.get("model") + if _needs_responses_surface(request_data) or not isinstance(model, str): + return None + if not has_native_responses_config(model, request_data): + return None + return native_logging_obj(logging_obj, request_data, request_data) + + +def _logged_by_client_stream(_completed: object) -> None: + return None + + +def _hand_completion_to_client_stream(result: object, executor_logging: "LiteLLMLoggingObj | None") -> None: + """The chat stream wrapping the executor's Responses stream logs the request once, in the shape the client got.""" + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + if executor_logging is not None and isinstance(result, BaseResponsesAPIStreamingIterator): + result.hand_completion_to(_logged_by_client_stream) + + +def _retagged_as_responses( + request_data: Mapping[str, object], logging_obj: "LiteLLMLoggingObj" +) -> Mapping[str, object]: + """`responses()` logs the Responses object it returns, so the shared logging object takes its call type.""" + from litellm.types.utils import CallTypes + + setattr(logging_obj, "call_type", CallTypes.responses.value) + return request_data + + class ResponsesToCompletionBridgeHandler: def __init__(self): from .transformation import LiteLLMResponsesTransformationHandler @@ -172,6 +228,7 @@ class ResponsesToCompletionBridgeHandler: from litellm import responses from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.responses.bridge_executor import native_responses validated_kwargs: Final = self.validate_input_kwargs(kwargs) model: Final = validated_kwargs["model"] @@ -199,11 +256,16 @@ class ResponsesToCompletionBridgeHandler: # `sanitized_litellm_params` already carries both, so passing them again # would raise a duplicate-keyword TypeError. request_data["custom_llm_provider"] = custom_llm_provider - request_data["model"] = _restore_routing_prefix(model, custom_llm_provider) + routed_model: Final = _restore_routing_prefix(model, custom_llm_provider) + request_data["model"] = routed_model if kwargs.get("cache") is not None: request_data["cache"] = kwargs["cache"] - result: Final = responses( - **request_data, + bridged_request: Final = _BRIDGED_REQUEST.validate_python(request_data) + executor_logging: Final = _executor_logging_obj(bridged_request, logging_obj) + result: Final = ( + responses(**_retagged_as_responses(request_data, logging_obj)) + if executor_logging is None + else native_responses(bridged_request, routed_model, bridged_request, executor_logging) ) from litellm.types.utils import ModelResponse @@ -253,6 +315,7 @@ class ResponsesToCompletionBridgeHandler: return self._apply_post_stream_processing(result, model, custom_llm_provider) from .transformation import is_structured_output_request + _hand_completion_to_client_stream(result, executor_logging) completion_stream: Final = self.transformation_handler.get_model_response_iterator( streaming_response=result, sync_stream=True, @@ -273,6 +336,7 @@ class ResponsesToCompletionBridgeHandler: async def acompletion(self, *args, **kwargs) -> Union["ModelResponse", "CustomStreamWrapper"]: from litellm import aresponses from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.responses.bridge_executor import anative_responses validated_kwargs: Final = self.validate_input_kwargs(kwargs) model: Final = validated_kwargs["model"] @@ -302,12 +366,16 @@ class ResponsesToCompletionBridgeHandler: # `sanitized_litellm_params` already carries both, so passing them again # would raise a duplicate-keyword TypeError. request_data["custom_llm_provider"] = custom_llm_provider - request_data["model"] = _restore_routing_prefix(model, custom_llm_provider) + routed_model: Final = _restore_routing_prefix(model, custom_llm_provider) + request_data["model"] = routed_model if kwargs.get("cache") is not None: request_data["cache"] = kwargs["cache"] - result: Final = await aresponses( - **request_data, - aresponses=True, + bridged_request: Final = _BRIDGED_REQUEST.validate_python(request_data) + executor_logging: Final = _executor_logging_obj(bridged_request, logging_obj) + result: Final = ( + await aresponses(**_retagged_as_responses(request_data, logging_obj), aresponses=True) + if executor_logging is None + else await anative_responses(bridged_request, routed_model, bridged_request, executor_logging) ) from litellm.types.utils import ModelResponse @@ -357,6 +425,7 @@ class ResponsesToCompletionBridgeHandler: return self._apply_post_stream_processing(result, model, custom_llm_provider) from .transformation import is_structured_output_request + _hand_completion_to_client_stream(result, executor_logging) completion_stream: Final = self.transformation_handler.get_model_response_iterator( streaming_response=result, sync_stream=False, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index bb59b4617af..25e63a7134a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1195,10 +1195,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): api_model: Final = model - from litellm.types.utils import CallTypes - - setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) - sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params) request_data: Final = { diff --git a/litellm/images/main.py b/litellm/images/main.py index 165543aadba..f47e56597a3 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -41,13 +41,13 @@ from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_ge from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config from litellm.main import ( azure_chat_completions, - base_llm_aiohttp_handler, base_llm_http_handler, bedrock_image_edit, bedrock_image_generation, openai_chat_completions, openai_image_variations, ) +from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler ########################################### from litellm.secret_managers.main import get_secret_str diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4dd33fa5565..26977a493ed 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -79,6 +79,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, is_expected_client_error, native_messages_reported_cost, + process_response_headers, provider_reported_usage_cost, proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, @@ -646,6 +647,10 @@ class Logging(LiteLLMLoggingBaseClass): baseline_observation: "CapturedBaselineObservation | None" = None _on_failure_payload_built: "Callable[[], None] | None" = None + def hold_stream_completion(self, completion: tuple[object, ...]) -> None: + """Parks stream-end logging for the proxy to release once its end-of-stream guardrails finish.""" + setattr(self, "_deferred_stream_complete_args", completion) # noqa: B010 # declaring the slot here turns the proxy's reads of it into private-usage findings + @property def _defer_async_logging(self) -> bool: return self.defer_async_logging @@ -4950,6 +4955,29 @@ class Logging(LiteLLMLoggingBaseClass): row is built from this response rather than re-parsed from the served body.""" self.bridged_model_response = response + def price_bridged_provider_response(self, response: ModelResponse | ResponsesAPIResponse) -> None: + """Stamps the cost and headers of the provider response a bridge translates into its client's shape. + + The client shape has no field for some of the usage the price depends on (built-in tool calls, + the 1h cache-write split), and the translation carries the stamped cost over, so the request is + billed for what the provider returned. The headers get the round of processing the call that made + the response gave them, which also names the provider request id `llm_provider-request-id`.""" + set_hidden_param(response, "response_cost", self.response_cost_calculator(result=response)) + provider_headers: Final = (get_hidden_params(response) or {}).get("additional_headers") + if isinstance(provider_headers, dict): + set_hidden_param( + response, + "additional_headers", + process_response_headers(provider_headers, preserve_litellm_internal_headers=True), + ) + + def price_bridged_provider_stream(self, stream: object) -> None: + """Prices a provider stream a bridge wraps for its client, as the call that opened it did. + + Nothing has streamed yet, so this only records the zero cost breakdown the proxy builds the + response headers from; the request is billed when the stream ends.""" + self.response_cost_calculator(result=stream) + def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None: self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response)) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 5542249d2c7..708c527e452 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -305,6 +305,7 @@ def _provider_hidden_params( class CustomStreamWrapper: _earlier_rounds_spend: "BilledRoundsSpend | None" = None + _on_complete: Callable[[object], None] | None = None @property def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation @@ -1755,7 +1756,7 @@ class CustomStreamWrapper: setattr(e, "message", str(e)) raise self._map_stream_exception(e) - def set_logging_event_loop(self, loop): + def set_logging_event_loop(self, loop: asyncio.AbstractEventLoop) -> None: """ import litellm, asyncio @@ -1907,11 +1908,19 @@ class CustomStreamWrapper: if not cache_hit and self.logging_obj.llm_caching_handler is not None: await self.logging_obj.llm_caching_handler.add_streaming_response_to_cache(processed_chunk) + def hand_completion_to(self, on_complete: Callable[[object], None]) -> None: + """A bridge whose own client stream logs the request takes the assembled response instead of the logger.""" + self._on_complete = on_complete + + def take_completion_back(self) -> None: + """The logger gets the assembled response again, as before a bridge took it.""" + self._on_complete = None + def run_success_logging_and_cache_storage(self, processed_chunk, cache_hit: bool): """ Runs success logging in a thread and adds the response to the cache """ - if litellm.disable_streaming_logging is True: + if litellm.disable_streaming_logging is True or self._on_complete is not None: """ [NOT RECOMMENDED] Set this via `litellm.disable_streaming_logging = True`. @@ -2174,13 +2183,18 @@ class CustomStreamWrapper: processed_chunk=_cache_copy, cache_hit=cache_hit, ) - executor.submit( - self.logging_obj.success_handler, - _log_copy, - None, - None, - cache_hit, - ) + if self._on_complete is not None: + self._on_complete(_log_copy) + else: + executor.submit( + self.logging_obj.success_handler, + _log_copy, + None, + None, + cache_hit, + ) + elif self._on_complete is not None: + self._on_complete(response) else: executor.submit( self.logging_obj.success_handler, @@ -2415,7 +2429,9 @@ class CustomStreamWrapper: "_on_deferred_stream_complete", None, ) - if _deferred_cb is not None: + if self._on_complete is not None: + self._on_complete(complete_streaming_response) + elif _deferred_cb is not None: # Proxy has post-call guardrails. Store the assembled # response so the outer streaming consumer # (ProxyLogging.async_post_call_streaming_iterator_hook) @@ -2423,10 +2439,7 @@ class CustomStreamWrapper: # end-of-stream blocks complete. Scheduling here via # create_task would race with unified_guardrail's # end-of-stream block for short-stream providers. - self.logging_obj._deferred_stream_complete_args = ( - complete_streaming_response, - cache_hit, - ) + self.logging_obj.hold_stream_completion((complete_streaming_response, cache_hit)) else: # prefer_async_handlers routes CustomLogger to async_success_handler # when consumers use ``async for`` on sync-SDK streams. Legacy string diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 704bec9056f..50294af9ee2 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -24,6 +24,7 @@ from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text +from litellm.llms.anthropic.messages.held_stream_completion import hold_completion_of from litellm.llms.anthropic.messages.safeguards import ( SafeguardsEvaluator, build_safeguards_evaluator, @@ -39,6 +40,7 @@ from litellm.llms.anthropic.messages.utils import ( proxy_sub_call_logging_kwargs, proxy_sub_call_trace_id, ) +from litellm.native.chat_plan import aexecute_planned_chat, execute_planned_chat, plan_chat from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -797,8 +799,13 @@ class LiteLLMMessagesToCompletionTransformationHandler: extra_kwargs=kwargs, ) - completion_response: Final = await litellm.acompletion(**completion_kwargs) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) + planned_chat: Final = None if logging_obj is None else plan_chat(completion_kwargs, logging_obj) + completion_response: Final = ( + await litellm.acompletion(**completion_kwargs) + if planned_chat is None + else await aexecute_planned_chat(planned_chat) + ) if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( @@ -810,6 +817,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: completion_request=completion_kwargs, litellm_logging_obj=logging_obj, safeguards_evaluator=safeguards_evaluator, + held_completion=None if planned_chat is None else hold_completion_of(completion_response, logging_obj), ) if transformed_stream is not None: return transformed_stream @@ -944,8 +952,11 @@ class LiteLLMMessagesToCompletionTransformationHandler: extra_kwargs=kwargs, ) - completion_response: Final = litellm.completion(**completion_kwargs) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) + planned_chat: Final = None if logging_obj is None else plan_chat(completion_kwargs, logging_obj) + completion_response: Final = ( + litellm.completion(**completion_kwargs) if planned_chat is None else execute_planned_chat(planned_chat) + ) if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( @@ -956,6 +967,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: is_async=False, completion_request=completion_kwargs, litellm_logging_obj=logging_obj, + held_completion=None if planned_chat is None else hold_completion_of(completion_response, logging_obj), ) if transformed_stream is not None: return transformed_stream diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index b19c0d8e2a9..6070287a3de 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -53,6 +53,7 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDe if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject + from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.utils import ModelResponseStream @@ -637,6 +638,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, completion_request: Mapping[str, object] | None = None, + held_completion: "HeldStreamCompletion | None" = None, ): # Wrap the upstream stream so chunks that carry both content and a # finish_reason (fake-streamed providers) are split into two — see @@ -651,6 +653,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id) self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator + self._held_completion: Final = held_completion # Mapping of truncated tool names to original names (for OpenAI's 64-char limit) self.tool_name_mapping = tool_name_mapping or {} # Polyfill applied_edits on final message_delta. @@ -1350,14 +1353,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): This wrapper ensures dict chunks are SSE formatted with both event and data lines. """ - for chunk in self: - if isinstance(chunk, dict): - event_type: str = str(chunk.get("type", "message")) - payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" - yield payload.encode() - else: - # For non-dict chunks, forward the original value unchanged - yield chunk + try: + for chunk in self: + if isinstance(chunk, dict): + event_type: str = str(chunk.get("type", "message")) + payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" + yield payload.encode() + else: + # For non-dict chunks, forward the original value unchanged + yield chunk + finally: + if self._held_completion is not None: + self._held_completion.log(is_async=False) async def aclose(self) -> None: await self._combined_chunk_splitter.aclose() @@ -1379,6 +1386,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): else: yield anthropic_sse_frame(await safeguard_results.observe(_SSE_EVENT.validate_python(chunk))) except Exception as e: # noqa: BLE001 # boundary before the socket: any upstream failure becomes an Anthropic error event + if self._held_completion is not None: + self._held_completion.discard() verbose_logger.exception("Anthropic Adapter - mid-stream error: %s", e) logging_obj: Final = self.litellm_logging_obj provider_error: Final = _provider_error(e) @@ -1402,6 +1411,12 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): failure_handler_error, ) yield _mid_stream_error_sse_event(e) + else: + if self._held_completion is not None: + self._held_completion.log(is_async=True) + finally: + if self._held_completion is not None: + self._held_completion.log(is_async=True, client_detached=True) def _increment_content_block_index(self): self.current_content_block_index += 1 diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index ddda759384f..a91cdb34637 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -213,6 +213,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject + from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.anthropic import ContentBlockContentBlockDict @@ -326,6 +327,7 @@ class AnthropicAdapter: litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, completion_request: Mapping[str, object] | None = None, + held_completion: "HeldStreamCompletion | None" = None, ) -> AsyncIterator[bytes] | Iterator[bytes] | None: """ Translate OpenAI streaming response to Anthropic format. @@ -355,6 +357,7 @@ class AnthropicAdapter: litellm_logging_obj=litellm_logging_obj, safeguards_evaluator=safeguards_evaluator, completion_request=completion_request, + held_completion=held_completion, ) # Return the SSE-wrapped version for proper event formatting. if is_async: @@ -1620,7 +1623,8 @@ class LiteLLMAnthropicMessagesAdapter: else translated_finish_reason ) # extract usage - usage: Final[Usage] = getattr(response, "usage") + # A provider reply with no usage block leaves the response without one; it reads as zero tokens. + usage: Final[Usage] = getattr(response, "usage", None) or Usage() message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage) polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None anthropic_usage: Final[AnthropicUsage] = ( diff --git a/litellm/llms/anthropic/messages/held_stream_completion.py b/litellm/llms/anthropic/messages/held_stream_completion.py new file mode 100644 index 00000000000..8b84fd6df3e --- /dev/null +++ b/litellm/llms/anthropic/messages/held_stream_completion.py @@ -0,0 +1,64 @@ +from collections.abc import Callable +from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable + +from litellm.responses.streaming_iterator import dispatch_stream_success + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + +@runtime_checkable +class _HandsOffCompletion(Protocol): + def hand_completion_to(self, on_complete: Callable[[object], None]) -> None: ... + + +class _NothingHeld: + pass + + +_NOTHING_HELD: Final = _NothingHeld() + + +class HeldStreamCompletion: + """The executor stream's assembled response, logged by the client-facing Messages stream when it ends. + + A stream that ended with no content hands over ``None``; it is still logged, as the success + handler logs an empty stream when it is not held. + """ + + def __init__(self, logging_obj: "LiteLLMLoggingObj") -> None: + self._logging_obj: Final = logging_obj + self._response: object = _NOTHING_HELD + + def take(self, response: object) -> None: + self._response = response + + def _release(self) -> object: + response: Final = self._response + self._response = _NOTHING_HELD + return response + + def discard(self) -> None: + self._release() + + def log(self, *, is_async: bool, client_detached: bool = False) -> None: + response: Final = self._release() + if response is _NOTHING_HELD: + return + dispatch_stream_success( + self._logging_obj, + response, + start_time=None, + end_time=None, + cache_hit=None, + is_async=is_async, + client_detached=client_detached, + ) + + +def hold_completion_of(stream: object, logging_obj: "LiteLLMLoggingObj | None") -> HeldStreamCompletion | None: + if logging_obj is None or not isinstance(stream, _HandsOffCompletion): + return None + held: Final = HeldStreamCompletion(logging_obj) + stream.hand_completion_to(held.take) + return held diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..e4a73583ca8 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -16,7 +16,6 @@ from pydantic import TypeAdapter import litellm from litellm.anthropic_beta_headers_manager import aload_beta_headers_config from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( @@ -29,23 +28,17 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.native.messages import execute_messages +from litellm.native.route_plan import NativeChat, NativeResponses, native_messages_config, plan_messages_route from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway -from litellm.router_utils.reasoning_effort_capability import ( - RequestedReasoningEffort, - deployment_reasoning_effort_levels, - keeps_effort, - model_group_from_metadata, - resolve_deployment_reasoning_effort, - with_adaptive_thinking_as_effort, -) from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, client +from litellm.types.utils import CallTypes +from litellm.utils import client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler @@ -56,69 +49,14 @@ from ..safeguards import ( with_safeguard_results, ) from .interceptors import get_messages_interceptors -from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages +from .utils import mock_response, prepare_native_messages __all__ = ("anthropic_messages", "anthropic_messages_handler") -# Providers that are routed directly to the OpenAI Responses API instead of -# going through chat/completions. -_RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) -def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool: - from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping - - model_info, _ = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - deployment_model_info=str_keyed_mapping(deployment_model_info), - ) - return model_info.get("mode") == "responses" - - -def _responses_mode_is_lost_by_prefix_strip( - requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object -) -> bool: - """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped. - - ``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a - deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the - chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions. - """ - if requested_model == resolved_model: - return False - return _bridges_to_responses_api( - requested_model, custom_llm_provider, deployment_model_info - ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) - - -def _should_route_to_responses_api( - custom_llm_provider: str | None, - requested_model: str | None = None, - resolved_model: str | None = None, - deployment_model_info: object = None, - stop_sequences: Sequence[str] | None = None, -) -> bool: - """Return True when the request should use the Responses API path. - - Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to - opt out and route OpenAI/Azure requests through chat/completions instead. - """ - if litellm.use_chat_completions_url_for_anthropic_messages: - return False - if stop_sequences: - return False - if custom_llm_provider in _RESPONSES_API_PROVIDERS: - return True - if custom_llm_provider is None or requested_model is None or resolved_model is None: - return False - return _responses_mode_is_lost_by_prefix_strip( - requested_model, resolved_model, custom_llm_provider, deployment_model_info - ) - - async def _with_native_safeguard_results( pending: Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]], proxy_classifies: Callable[[], Awaitable[bool]], @@ -146,88 +84,6 @@ def _native_safeguards_evaluator(kwargs: Mapping[str, object], messages: object) return request_safeguards_evaluator(kwargs, _SAFEGUARD_MESSAGES.validate_python(messages)) -def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: - """Whether the deployment opted into forwarding /v1/messages untranslated. - - The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``, - declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]`` - by the router. - """ - if not isinstance(model_info, dict): - return False - supported_endpoints: Final = model_info.get("supported_endpoints") - return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints - - -def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]: - rest: Final = {key: value for key, value in params.items() if key != "output_config"} - return rest if output_config is None else {**rest, "output_config": output_config} - - -def _with_deployment_output_effort( - params: Mapping[str, object], - *, - model: str, - custom_llm_provider: str, - kwargs: Mapping[str, object], - drop_params: bool, -) -> Mapping[str, object]: - deployment_model_info: Final = kwargs.get("model_info") - - def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort: - return resolve_deployment_reasoning_effort( - requested, - model=model, - custom_llm_provider=custom_llm_provider, - deployment_model_info=deployment_model_info, - model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")), - drop_params=drop_params, - ).value - - def translate(source: Mapping[str, object]) -> Mapping[str, object]: - return with_adaptive_thinking_as_effort( - source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params - ) - - requested_output: Final = as_str_mapping(params.get("output_config")) - caller_output: Final = None if requested_output is None else resolve(requested_output) - caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output) - if deployment_reasoning_effort_levels(deployment_model_info) is None: - return caller_params - if requested_output is not None and keeps_effort(requested_output, caller_output): - return _with_output_config(translate(params), caller_output) - translated: Final = translate(caller_params) - budget_output: Final = as_str_mapping(translated.get("output_config")) - return translated if budget_output is None else _with_output_config(translated, resolve(budget_output)) - - -def _deployment_supports_cache_control_ttl(model_info: object) -> bool: - return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True - - -def _native_messages_config( - model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None -) -> BaseAnthropicMessagesConfig | None: - provider_config: Final = ( - ProviderConfigManager.get_provider_anthropic_messages_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), - ) - if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders] - else None - ) - native_config: Final = ( - provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None - ) - if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info): - return native_config - from litellm.llms.openai_like.messages.transformation import ( - OpenAILikeAnthropicMessagesConfig, - ) - - return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info)) - - async def _proxy_classifies(model: str, safeguards: object, native_config: BaseAnthropicMessagesConfig | None) -> bool: if not proxy_serves_safeguards(safeguards): return False @@ -244,7 +100,7 @@ async def proxy_classifies_tool_uses(model: str, custom_llm_provider: str | None return await _proxy_classifies( model, kwargs.get("safeguards"), - _native_messages_config( + native_messages_config( model, custom_llm_provider, kwargs.get("model_info"), api_base if isinstance(api_base, str) else None ), ) @@ -687,40 +543,44 @@ def anthropic_messages_handler( **kwargs, ) - anthropic_messages_provider_config: Final = _native_messages_config( - model, custom_llm_provider, kwargs.get("model_info"), api_base + route: Final = plan_messages_route( + requested_model=original_model, + resolved_model=model, + custom_llm_provider=custom_llm_provider, + model_info=kwargs.get("model_info"), + api_base=api_base, + compaction_requested=kwargs.get("compaction") is not None, + force_chat_completions=bool(litellm.use_chat_completions_url_for_anthropic_messages), + stop_requested=bool(stop_sequences), ) - if anthropic_messages_provider_config is None: - # Route to Responses API for OpenAI / Azure, chat/completions for everything else. - if kwargs.get("compaction") is None and _should_route_to_responses_api( - custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences - ): - return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( - max_tokens=max_tokens, - messages=messages, - model=original_model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower - "str | None", system - ), - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=cast( # cast-ok: dict tools already fit the handler's union - "list[AllAnthropicToolsValues | dict[str, object]] | None", tools - ), - top_k=top_k, - top_p=top_p, - _is_async=is_async, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, - ) + if isinstance(route, NativeResponses): + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=max_tokens, + messages=messages, + model=original_model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower + "str | None", system + ), + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=cast( # cast-ok: dict tools already fit the handler's union + "list[AllAnthropicToolsValues | dict[str, object]] | None", tools + ), + top_k=top_k, + top_p=top_p, + _is_async=is_async, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + if isinstance(route, NativeChat): # The in-gateway context_management polyfill runs inside # ``async_anthropic_messages_handler`` so it can ``await`` the # summarization model for ``compact_20260112``. ``context_management`` @@ -761,23 +621,11 @@ def anthropic_messages_handler( if marked_tools is not None: AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) - anthropic_messages_optional_request_params: Final = ( - AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( - params=local_vars, - model=model, - drop_params=litellm_params.get("drop_params") is True, - custom_llm_provider=custom_llm_provider, - ) - ) - resolved_api_base: Final = ( - dynamic_api_base - if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base() - else api_base - ) safeguard_kwargs: Final = ( _SAFEGUARD_KWARGS.validate_python(kwargs) if is_async is True and "safeguards" in kwargs else None ) - native_response: Final = base_llm_http_handler.anthropic_messages_handler( + native_response: Final = execute_messages( + http_handler=base_llm_http_handler, model=model, messages=cast( # cast-ok: anthropic messages are dicts at runtime "list[dict[str, object]]", @@ -785,30 +633,23 @@ def anthropic_messages_handler( cast(Sequence[object], messages) # cast-ok: anthropic payloads arrive as untyped dicts ), ), - anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=dict( - _with_deployment_output_effort( - anthropic_messages_optional_request_params, - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=cast(Mapping[str, object], kwargs), # cast-ok: router kwargs arrive untyped - drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True, - ) - ), - _is_async=is_async, - client=client, + provider_config=route.config, + request_params=local_vars, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, api_key=api_key, - api_base=resolved_api_base, + api_base=api_base, + dynamic_api_base=dynamic_api_base, + client=client, + is_async=is_async, stream=stream, - kwargs=kwargs, + request_kwargs=kwargs, ) if safeguard_kwargs is None or not isinstance(native_response, Coroutine): return native_response return _with_native_safeguard_results( native_response, - partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), anthropic_messages_provider_config), + partial(_proxy_classifies, model, safeguard_kwargs.get("safeguards"), route.config), partial(_native_safeguards_evaluator, safeguard_kwargs, messages), ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 0eef3fd87c0..a63b56947f6 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -19,11 +19,14 @@ from litellm.litellm_core_utils.asyncify import ( run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) +from litellm.llms.anthropic.messages.held_stream_completion import hold_completion_of +from litellm.responses.bridge_executor import anative_responses, native_logging_obj, native_responses from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -38,7 +41,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import CountContextManagement +from litellm.types.utils import CallTypes, CountContextManagement from litellm.utils import ProviderConfigManager from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator @@ -159,6 +162,9 @@ def _with_applied_edits( return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))} +_NEVER_FORWARDED: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"} + + def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None: logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs) if logging_obj is not None: @@ -166,8 +172,10 @@ def _record_bridged_response(responses_kwargs: Mapping[str, object], result: Res def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]: - """The litellm-specific kwargs forwarded verbatim onto the Responses API request.""" - return extra_kwargs or {} + """The litellm-specific kwargs (credentials, routing, metadata) that carry over to the Responses API call.""" + return { + key: value for key, value in (extra_kwargs or {}).items() if key not in _NEVER_FORWARDED and value is not None + } def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool: @@ -247,7 +255,7 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin return () -def _build_responses_kwargs( +def _build_responses_request( *, max_tokens: int, messages: AnthropicRequestMessages, @@ -267,9 +275,7 @@ def _build_responses_kwargs( output_format: AnthropicOutputSchema | None = None, extra_kwargs: Mapping[str, object] | None = None, ) -> dict[str, object]: - """ - Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). - """ + """The Responses API request the Messages request translates to, without the forwarded litellm kwargs.""" forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) reject_malformed_tool_result_text(messages, model, forwarded_kwargs) block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) @@ -314,7 +320,7 @@ def _build_responses_kwargs( request_data["output_format"] = output_format anthropic_request: Final = AnthropicMessagesRequest(**request_data) - responses_kwargs: Final = _ADAPTER.translate_request( + responses_request: Final = _ADAPTER.translate_request( anthropic_request, include_encrypted_reasoning=_provider_returns_encrypted_reasoning( model, forwarded_kwargs.get("custom_llm_provider") @@ -325,7 +331,7 @@ def _build_responses_kwargs( # Normalize reasoning effort based on model capabilities # (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported) - reasoning: Final = responses_kwargs.get("reasoning") + reasoning: Final = responses_request.get("reasoning") if isinstance(reasoning, dict): effort: Final[object] = reasoning.get("effort") if isinstance(effort, str): @@ -341,49 +347,39 @@ def _build_responses_kwargs( deployment_model_info=forwarded_kwargs.get("model_info"), ) if normalized != effort: - responses_kwargs["reasoning"] = {**reasoning, "effort": normalized} + responses_request["reasoning"] = {**reasoning, "effort": normalized} if stream: - responses_kwargs["stream"] = True - - # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) - excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"} - for key, value in forwarded_kwargs.items(): - if key == "litellm_logging_obj" and value is not None: - from litellm.litellm_core_utils.litellm_logging import ( - Logging as LiteLLMLoggingObject, - ) - from litellm.types.utils import CallTypes - - if isinstance(value, LiteLLMLoggingObject): - # Keep call_type as anthropic_messages so spend_logs are billed - # against /v1/messages; the success handler translates the - # Responses API result back to a ModelResponse for the row. - setattr(value, "call_type", CallTypes.anthropic_messages.value) - responses_kwargs[key] = value - elif key not in excluded and key not in responses_kwargs and value is not None: - responses_kwargs[key] = value + responses_request["stream"] = True explicit_prompt_cache_key: Final = forwarded_kwargs.get("prompt_cache_key") if explicit_prompt_cache_key is not None: - responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key + responses_request["prompt_cache_key"] = explicit_prompt_cache_key deployment_include: Final = forwarded_kwargs.get("include") - bridge_include: Final = responses_kwargs.get("include") + bridge_include: Final = responses_request.get("include") if isinstance(deployment_include, list) and isinstance(bridge_include, list): - responses_kwargs["include"] = [ + responses_request["include"] = [ *bridge_include, *(item for item in deployment_include if item not in bridge_include), ] - return responses_kwargs + return responses_request + + +def _reentrant_kwargs( + forwarded_kwargs: Mapping[str, object], request: Mapping[str, object], logging_obj: LiteLLMLoggingObj | None +) -> Mapping[str, object]: + if logging_obj is not None: + setattr(logging_obj, "call_type", CallTypes.anthropic_messages.value) + return {**forwarded_kwargs, **request} class LiteLLMMessagesToResponsesAPIHandler: """ Handles Anthropic /v1/messages requests for OpenAI / Azure models by - calling litellm.responses() / litellm.aresponses() directly and translating - the response back to Anthropic format. + calling the native Responses executor and translating the response back + to Anthropic format. """ @staticmethod @@ -410,7 +406,7 @@ class LiteLLMMessagesToResponsesAPIHandler: polyfill_result: Final = await _polyfilled( model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs ) - responses_kwargs: Final = _build_responses_kwargs( + request: Final = _build_responses_request( max_tokens=max_tokens, messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, @@ -429,17 +425,25 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format=output_format, extra_kwargs=kwargs, ) + forwarded_kwargs: Final = _forwarded_kwargs(kwargs) + logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) - result: Final = await litellm.aresponses(**responses_kwargs) + native_logging: Final = native_logging_obj(logging_obj, forwarded_kwargs, request) + result: Final = ( + await litellm.aresponses(**_reentrant_kwargs(forwarded_kwargs, request, logging_obj)) + if native_logging is None + else await anative_responses(request, model, forwarded_kwargs, native_logging) + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")), - litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), + litellm_logging_obj=logging_obj, safeguards_evaluator=safeguards_evaluator, applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, + held_completion=None if native_logging is None else hold_completion_of(result, native_logging), ) return wrapper.async_anthropic_sse_wrapper() @@ -449,7 +453,7 @@ class LiteLLMMessagesToResponsesAPIHandler: served: Final = await with_safeguard_results( _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator ) - _record_bridged_response(responses_kwargs, result) + _record_bridged_response(kwargs, result) return served @staticmethod @@ -514,7 +518,7 @@ class LiteLLMMessagesToResponsesAPIHandler: else None ) polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None - responses_kwargs: Final = _build_responses_kwargs( + request: Final = _build_responses_request( max_tokens=max_tokens, messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, @@ -533,15 +537,22 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format=output_format, extra_kwargs=kwargs, ) - - result: Final = litellm.responses(**responses_kwargs) + forwarded_kwargs: Final = _forwarded_kwargs(kwargs) + logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) + native_logging: Final = native_logging_obj(logging_obj, forwarded_kwargs, request) + result: Final = ( + litellm.responses(**_reentrant_kwargs(forwarded_kwargs, request, logging_obj)) + if native_logging is None + else native_responses(request, model, forwarded_kwargs, native_logging) + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")), - litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), + litellm_logging_obj=logging_obj, applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, + held_completion=None if native_logging is None else hold_completion_of(result, native_logging), ) return wrapper.async_anthropic_sse_wrapper() diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index dae96ba4ce1..4cf0c442115 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -38,6 +38,7 @@ from .transformation import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject + from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.openai import ResponseIncompleteEvent @@ -131,12 +132,14 @@ class AnthropicResponsesStreamWrapper: litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, applied_edits: Sequence[AppliedEdit] | None = None, + held_completion: "HeldStreamCompletion | None" = None, ) -> None: self.responses_stream = responses_stream self._applied_edits: Final = tuple(applied_edits or ()) self.model = model self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator + self._held_completion: Final = held_completion self._message_id: str = f"msg_{uuid.uuid4()}" if litellm_logging_obj is not None: litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id) @@ -545,10 +548,22 @@ class AnthropicResponsesStreamWrapper: async def _sse_frames(self) -> AsyncGenerator[bytes, None]: safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator) + held: Final = self._held_completion try: async for chunk in self: yield anthropic_sse_frame(await safeguard_results.observe(chunk)) + except Exception: + if held is not None: + held.discard() + raise + else: + if held is not None and self._stream_failed: + held.discard() + if held is not None: + held.log(is_async=True) finally: + if held is not None: + held.log(is_async=True, client_detached=True) from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt if not closing_superseded_attempt(): diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index a1ba8de6d7f..c25944b967b 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -1,7 +1,7 @@ import asyncio import json import time -from collections.abc import Callable, Coroutine +from collections.abc import Callable, Coroutine, Mapping from typing import Final import httpx @@ -114,7 +114,7 @@ class AzureOpenAIAssistantsAPIConfig: def check_dynamic_azure_params( - azure_client_params: dict, + azure_client_params: Mapping[str, object], azure_client: AzureOpenAI | AsyncAzureOpenAI | None, ) -> bool: """ diff --git a/litellm/main.py b/litellm/main.py index d469aa629fe..c9bdb345e07 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -14,18 +14,15 @@ import contextvars import datetime import inspect import json -import os import random import sys -import time import traceback -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mapping, Sequence +from collections.abc import AsyncIterator, Coroutine, Mapping from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy from functools import partial -from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast from litellm._logging import redact_string from litellm._uuid import uuid @@ -37,35 +34,35 @@ import dotenv import httpx import openai from pydantic import BaseModel, TypeAdapter, ValidationError -from typing_extensions import assert_never, overload +from typing_extensions import overload import litellm # client must be imported from litellm as it's a decorator used at function definition time from litellm import client -from litellm.types.llms.base import LiteLLMBaseModel # Other utils are imported directly to avoid circular imports -from litellm.utils import ( - exception_type, - filter_out_litellm_params, - get_litellm_params, - get_optional_params, - peek_reasoning_summary_aliases, - reasoning_effort_with_summary_alias, - strip_reasoning_summary_aliases_from_optional_params, -) +from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params if TYPE_CHECKING: from litellm.router import Router from litellm.types.utils import TokenCountResponse +import os as os # noqa: PLC0414 # litellm.* re-export + +from typing_extensions import ( + assert_never as assert_never, # noqa: PLC0414 # litellm.* re-export +) + +from litellm.constants import AZURE_OPENAI_AUDIO_PROVIDERS, OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS from litellm.constants import ( - AZURE_OPENAI_AUDIO_PROVIDERS, - DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - NADIR_DEFAULT_API_BASE, - OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS, + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT as DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, # noqa: PLC0414 # litellm.* re-export +) +from litellm.constants import ( + DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT as DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, # noqa: PLC0414 # litellm.* re-export +) +from litellm.constants import ( + NADIR_DEFAULT_API_BASE as NADIR_DEFAULT_API_BASE, # noqa: PLC0414 # litellm.* re-export ) from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger @@ -77,23 +74,32 @@ from litellm.litellm_core_utils.audio_utils.utils import ( from litellm.litellm_core_utils.chat_completion_agentic_loop import ( maybe_run_chat_completion_agentic_loop, ) -from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( - AWS_CREDENTIAL_KWARGS_KEYS, - OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, - OPTIONAL_KWARGS_KEYS, - PROVIDER_AFFINITY_HEADER_KWARG_KEY, - InvalidControlOption, - parse_control_options, - with_control_options, + AWS_CREDENTIAL_KWARGS_KEYS as AWS_CREDENTIAL_KWARGS_KEYS, # noqa: PLC0414 # litellm.* re-export +) +from litellm.litellm_core_utils.get_litellm_params import ( + OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS as OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, # noqa: PLC0414 # litellm.* re-export +) +from litellm.litellm_core_utils.get_litellm_params import OPTIONAL_KWARGS_KEYS +from litellm.litellm_core_utils.get_litellm_params import ( + PROVIDER_AFFINITY_HEADER_KWARG_KEY as PROVIDER_AFFINITY_HEADER_KWARG_KEY, # noqa: PLC0414 # litellm.* re-export +) +from litellm.litellm_core_utils.get_litellm_params import ( + InvalidControlOption as InvalidControlOption, # noqa: PLC0414 # litellm.* re-export +) +from litellm.litellm_core_utils.get_litellm_params import ( + parse_control_options as parse_control_options, # noqa: PLC0414 # litellm.* re-export +) +from litellm.litellm_core_utils.get_litellm_params import ( + with_control_options as with_control_options, # noqa: PLC0414 # litellm.* re-export ) from litellm.litellm_core_utils.get_llm_provider_logic import ( - get_provider_affinity_header, invalid_provider_routing_hints_message, ) from litellm.litellm_core_utils.get_provider_specific_headers import ( - ProviderSpecificHeaderUtils, + ProviderSpecificHeaderUtils as ProviderSpecificHeaderUtils, # noqa: PLC0414 # litellm.* re-export ) from litellm.litellm_core_utils.health_check_utils import ( _filter_model_params, @@ -108,40 +114,72 @@ from litellm.litellm_core_utils.mock_functions import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export ) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - messages_have_pdf_file_part, - messages_without_pdf_file_parts, +from litellm.litellm_core_utils.provider_affinity import ( + add_provider_affinity_header as add_provider_affinity_header, # noqa: PLC0414 # litellm.* re-export ) -from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.request_timeout_resolver import ( - get_configured_request_timeout, + get_configured_request_timeout as get_configured_request_timeout, # noqa: PLC0414 # litellm.* re-export ) from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer -from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig +from litellm.llms.base_llm import ( + BaseConfig as BaseConfig, # noqa: PLC0414 # litellm.* re-export +) from litellm.llms.base_llm.base_model_iterator import ( - convert_model_response_to_streaming, + convert_model_response_to_streaming as convert_model_response_to_streaming, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.base_llm.chat.transformation import ( + with_attribution_headers as with_attribution_headers, # noqa: PLC0414 # litellm.* re-export ) -from litellm.llms.base_llm.chat.transformation import with_attribution_headers from litellm.llms.bedrock.common_utils import ( - BedrockModelInfo, - NativeOutputFallback, - bedrock_route_for_request, - bedrock_strict_tools_fallback, - deployment_model_info, - without_bedrock_native_output_opt_in, - without_bedrock_route_prefix, + BedrockModelInfo as BedrockModelInfo, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock.common_utils import ( + bedrock_route_for_request as bedrock_route_for_request, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock.common_utils import ( + without_bedrock_route_prefix as without_bedrock_route_prefix, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.bedrock_mantle.chat.claude_transformation import ( + bedrock_mantle_chat_config as bedrock_mantle_chat_config, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.cohere.common_utils import ( + CohereModelInfo as CohereModelInfo, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, +) +from litellm.llms.custom_httpx.http_handler import ( + http2_enabled as http2_enabled, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.openai.chat.gpt_reasoning_transformation import ( + OpenAIGPT5Config as OpenAIGPT5Config, # noqa: PLC0414 # litellm.* re-export +) +from litellm.llms.openai_like.json_loader import ( + JSONProviderRegistry as JSONProviderRegistry, # noqa: PLC0414 # litellm.* re-export ) -from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config -from litellm.llms.cohere.common_utils import CohereModelInfo -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled -from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config -from litellm.llms.openai.common_utils import get_openai_credentials -from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( - VertexAIModelRoute, - get_vertex_ai_model_route, + VertexAIModelRoute as VertexAIModelRoute, # noqa: PLC0414 # litellm.* re-export ) +from litellm.llms.vertex_ai.common_utils import ( + get_vertex_ai_model_route as get_vertex_ai_model_route, # noqa: PLC0414 # litellm.* re-export +) +from litellm.native.chat import execute_chat +from litellm.native.chat import resolve_dispatched_chat_response as _resolve_dispatched_chat_response +from litellm.native.chat_mock import MOCK_RESPONSE_TYPE as MOCK_RESPONSE_TYPE # noqa: PLC0414 # litellm.* re-export +from litellm.native.chat_mock import handle_mock_timeout_async as _handle_mock_timeout_async +from litellm.native.chat_mock import mock_completion as mock_completion # noqa: PLC0414 # litellm.* re-export +from litellm.native.chat_providers.handlers import * # noqa: F403 # litellm.* re-export of the moved handlers +from litellm.native.chat_providers.support import print_verbose +from litellm.native.request_support import ( + admission_input_tokens as admission_input_tokens, # noqa: PLC0414 # litellm.* re-export +) +from litellm.native.request_support import ( + admitted_prompt_token_counter as admitted_prompt_token_counter, # noqa: PLC0414 # litellm.* re-export +) +from litellm.native.request_support import proxy_auth_headers as _proxy_auth_headers +from litellm.native.request_support import register_custom_pricing_for_request as _register_custom_pricing_for_request +from litellm.native.request_support import resolve_control_options as _resolve_control_options from litellm.realtime_api.main import realtime_health_check from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.router_utils.reasoning_effort_capability import ( @@ -149,26 +187,37 @@ from litellm.router_utils.reasoning_effort_capability import ( reasoning_effort_needs_allowing, resolve_chat_reasoning, ) -from litellm.secret_managers.main import get_secret_bool, get_secret_str -from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper -from litellm.types.completion import ( - CompletionDispatchContext, - _CompletionDispatchResult, +from litellm.secret_managers.main import ( + get_secret_bool as get_secret_bool, # noqa: PLC0414 # litellm.* re-export ) -from litellm.types.litellm_params import ControlOptions, RetryStrategy +from litellm.secret_managers.main import ( + get_secret_str, +) +from litellm.types.completion import ( + CompletionDispatchContext as CompletionDispatchContext, # noqa: PLC0414 # litellm.* re-export +) +from litellm.types.litellm_params import ( + ControlOptions as ControlOptions, # noqa: PLC0414 # litellm.* re-export +) +from litellm.types.litellm_params import RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( - AgenticLoopParams, BaseUrlRoutingHint, - CustomPricingLiteLLMParams, - ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, StreamingChoices, is_decisions_model_mode, ) -from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS +from litellm.types.utils import ( + ModelResponseStream as ModelResponseStream, # noqa: PLC0414 # litellm.* re-export +) +from litellm.types.workload_identity import ( + ANTHROPIC_WIF_KWARGS_KEYS as ANTHROPIC_WIF_KWARGS_KEYS, # noqa: PLC0414 # litellm.* re-export +) +from litellm.types.workload_identity import ( + OPENAI_WIF_KWARGS_KEYS as OPENAI_WIF_KWARGS_KEYS, # noqa: PLC0414 # litellm.* re-export +) from litellm.utils import ( Choices, CustomStreamWrapper, @@ -178,10 +227,9 @@ from litellm.utils import ( ProviderConfigManager, TextChoices, TextCompletionResponse, + TextCompletionStreamWrapper, TranscriptionResponse, - Usage, add_provider_specific_params_to_optional_params, - async_mock_completion_streaming_obj, convert_to_model_response_object, create_pretrained_tokenizer, create_tokenizer, @@ -192,22 +240,37 @@ from litellm.utils import ( get_optional_params_embeddings, get_optional_params_image_gen, get_optional_params_transcription, - get_requester_metadata, get_secret, get_standard_openai_params, - is_explicitly_disabled_factory, - mock_completion_streaming_obj, - pre_process_non_default_params, read_config_args, should_run_mock_completion, - supports_httpx_timeout, - supports_pdf_input, validate_and_fix_openai_messages, validate_and_fix_openai_tools, validate_and_fix_thinking_param, validate_chat_completion_tool_choice, validate_openai_optional_params, ) +from litellm.utils import ( + Usage as Usage, # noqa: PLC0414 # litellm.* re-export +) +from litellm.utils import ( + async_mock_completion_streaming_obj as async_mock_completion_streaming_obj, # noqa: PLC0414 # litellm.* re-export +) +from litellm.utils import ( + get_requester_metadata as get_requester_metadata, # noqa: PLC0414 # litellm.* re-export +) +from litellm.utils import ( + mock_completion_streaming_obj as mock_completion_streaming_obj, # noqa: PLC0414 # litellm.* re-export +) +from litellm.utils import ( + peek_reasoning_summary_aliases as peek_reasoning_summary_aliases, # noqa: PLC0414 # litellm.* re-export +) +from litellm.utils import ( + pre_process_non_default_params as pre_process_non_default_params, # noqa: PLC0414 # litellm.* re-export +) +from litellm.utils import ( + strip_reasoning_summary_aliases_from_optional_params as strip_reasoning_summary_aliases_from_optional_params, # noqa: PLC0414 # litellm.* re-export +) from litellm.utils import ( token_counter as token_counter, # noqa: PLC0414 # public re-export ) @@ -216,63 +279,56 @@ from ._logging import verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache from .litellm_core_utils.core_helpers import ( as_str_mapping, - drops_unsupported_params, normalize_drop_params, - safe_deep_copy, +) +from .litellm_core_utils.core_helpers import ( + safe_deep_copy as safe_deep_copy, # noqa: PLC0414 # litellm.* re-export ) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, completion_with_fallbacks, ) +from .litellm_core_utils.prompt_templates.common_utils import add_system_prompt_to_messages, get_completion_messages from .litellm_core_utils.prompt_templates.common_utils import ( - add_system_prompt_to_messages, - get_completion_messages, - update_messages_with_model_file_ids, + update_messages_with_model_file_ids as update_messages_with_model_file_ids, # noqa: PLC0414 # litellm.* re-export ) from .litellm_core_utils.prompt_templates.factory import ( - custom_prompt, - function_call_prompt, - map_system_message_pt, - ollama_pt, - prompt_factory, - stringify_json_tool_call_content, + function_call_prompt as function_call_prompt, # noqa: PLC0414 # litellm.* re-export +) +from .litellm_core_utils.prompt_templates.factory import ( + map_system_message_pt as map_system_message_pt, # noqa: PLC0414 # litellm.* re-export +) +from .litellm_core_utils.prompt_templates.factory import ( + stringify_json_tool_call_content as stringify_json_tool_call_content, # noqa: PLC0414 # litellm.* re-export ) from .litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor as ChunkProcessor, # noqa: PLC0414 # public re-export ) -from .llms.anthropic.chat import AnthropicChatCompletion -from .llms.anthropic.common_utils import is_claude_tool_target from .llms.azure.audio_transcriptions import AzureAudioTranscription -from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params -from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion -from .llms.azure.completion.handler import AzureTextCompletion -from .llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion -from .llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url +from .llms.azure.azure import ( + check_dynamic_azure_params as check_dynamic_azure_params, # noqa: PLC0414 # litellm.* re-export +) from .llms.azure_ai.embed import AzureAIEmbedding -from .llms.bedrock.chat import BedrockConverseLLM -from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from .llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import AmazonAnthropicClaudeConfig -from .llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback from .llms.bedrock.embed.embedding import BedrockEmbedding from .llms.bedrock.image_edit.handler import BedrockImageEdit from .llms.bedrock.image_generation.image_handler import BedrockImageGeneration -from .llms.bytez.chat.transformation import BytezChatConfig from .llms.clarifai.chat.transformation import ClarifaiConfig -from .llms.codestral.completion.handler import CodestralTextCompletion from .llms.cohere.embed import handler as cohere_embed -from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler -from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from .llms.custom_llm import CustomLLM, custom_chat_llm_router +from .llms.custom_llm import ( + CustomLLM, +) +from .llms.custom_llm import ( + custom_chat_llm_router as custom_chat_llm_router, # noqa: PLC0414 # litellm.* re-export +) from .llms.databricks.common_utils import DatabricksBase from .llms.databricks.embed.handler import DatabricksEmbeddingHandler -from .llms.deprecated_providers import aleph_alpha -from .llms.gdc.chat.transformation import GDCGeminiConfig +from .llms.deprecated_providers import ( + aleph_alpha as aleph_alpha, # noqa: PLC0414 # litellm.* re-export +) from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.huggingface.embedding.handler import HuggingFaceEmbedding -from .llms.lemonade.chat.transformation import LemonadeChatConfig -from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion from .llms.nvidia_riva.audio_transcription.handler import ( NvidiaRivaAudioTranscription, ) @@ -282,22 +338,15 @@ from .llms.nvidia_riva.audio_transcription.transformation import ( from .llms.oci.chat.transformation import OCIChatConfig from .llms.ollama.completion import handler as ollama from .llms.oobabooga.chat import oobabooga -from .llms.openai.completion.handler import OpenAITextCompletion from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler -from .llms.openai.openai import OpenAIChatCompletion from .llms.openai.transcriptions.handler import OpenAIAudioTranscription from .llms.openai.workload_identity import build_openai_client -from .llms.openai_like.chat.handler import OpenAILikeChatHandler from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler -from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig -from .llms.petals.completion import handler as petals_handler -from .llms.predibase.chat.handler import PredibaseChatCompletion -from .llms.replicate.chat.handler import completion as replicate_chat_completion from .llms.sagemaker.chat.handler import SagemakerChatHandler -from .llms.sagemaker.completion.handler import SagemakerLLM from .llms.sap.chat.handler import GenAIHubOrchestration -from .llms.vertex_ai import vertex_ai_non_gemini -from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from .llms.vertex_ai import ( + vertex_ai_non_gemini as vertex_ai_non_gemini, # noqa: PLC0414 # litellm.* re-export +) from .llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import ( GoogleBatchEmbeddings, ) @@ -307,12 +356,7 @@ from .llms.vertex_ai.image_generation.image_generation_handler import ( from .llms.vertex_ai.multimodal_embeddings.embedding_handler import ( VertexMultimodalEmbedding, ) -from .llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels from .llms.vertex_ai.vertex_embeddings.embedding_handler import VertexEmbedding -from .llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels -from .llms.vertex_ai.vertex_model_garden.main import VertexAIModelGardenModels -from .llms.vllm.completion import handler as vllm_handler -from .llms.watsonx.chat.handler import WatsonXChatHandler from .llms.watsonx.common_utils import IBMWatsonXMixin from .types.llms.anthropic import AnthropicThinkingParam from .types.llms.openai import ( @@ -327,195 +371,41 @@ from .types.llms.openai import ( ) from .types.utils import ( AdapterCompletionStreamWrapper, - ChatCompletionMessageToolCall, CompletionTokensDetails, FileTypes, HiddenParams, LlmProviders, PromptTokensDetails, - ProviderSpecificHeader, +) +from .types.utils import ( + ChatCompletionMessageToolCall as ChatCompletionMessageToolCall, # noqa: PLC0414 # litellm.* re-export +) +from .types.utils import ( + ProviderSpecificHeader as ProviderSpecificHeader, # noqa: PLC0414 # litellm.* re-export ) ####### ENVIRONMENT VARIABLES ################### -openai_chat_completions = OpenAIChatCompletion() -openai_text_completions: Final = OpenAITextCompletion() openai_audio_transcriptions: Final = OpenAIAudioTranscription() nvidia_riva_audio_transcriptions: Final = NvidiaRivaAudioTranscription() openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions: Final = GroqChatCompletion() -sap_gen_ai_hub_chat_completions: Final = GenAIHubOrchestration() sap_gen_ai_hub_emb: Final = GenAIHubOrchestration() azure_ai_embedding: Final = AzureAIEmbedding() -anthropic_chat_completions: Final = AnthropicChatCompletion() -azure_anthropic_chat_completions: Final = AzureAnthropicChatCompletion() -azure_chat_completions = AzureChatCompletion() -azure_o1_chat_completions: Final = AzureOpenAIO1ChatCompletion() -azure_text_completions: Final = AzureTextCompletion() azure_audio_transcriptions: Final = AzureAudioTranscription() huggingface_embed: Final = HuggingFaceEmbedding() -predibase_chat_completions: Final = PredibaseChatCompletion() -codestral_text_completions: Final = CodestralTextCompletion() -bedrock_converse_chat_completion: Final = BedrockConverseLLM() bedrock_embedding: Final = BedrockEmbedding() bedrock_image_generation = BedrockImageGeneration() bedrock_image_edit = BedrockImageEdit() -vertex_chat_completion: Final = VertexLLM() vertex_embedding: Final = VertexEmbedding() vertex_multimodal_embedding: Final = VertexMultimodalEmbedding() vertex_image_generation: Final = VertexImageGeneration() google_batch_embeddings: Final = GoogleBatchEmbeddings() -vertex_partner_models_chat_completion: Final = VertexAIPartnerModels() -vertex_gemma_chat_completion: Final = VertexAIGemmaModels() -vertex_model_garden_chat_completion: Final = VertexAIModelGardenModels() -gdc_transformation: Final = GDCGeminiConfig() # vertex_text_to_speech is now replaced by VertexAITextToSpeechConfig -sagemaker_llm: Final = SagemakerLLM() -watsonx_chat_completion: Final = WatsonXChatHandler() openai_like_embedding: Final = OpenAILikeEmbeddingHandler() -openai_like_chat_completion: Final = OpenAILikeChatHandler() databricks_embedding: Final = DatabricksEmbeddingHandler() -base_llm_http_handler = BaseLLMHTTPHandler() -base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() sagemaker_chat_completion: Final = SagemakerChatHandler() -bytez_transformation: Final = BytezChatConfig() heroku_transformation: Final = HerokuChatConfig() oci_transformation: Final = OCIChatConfig() -ovhcloud_transformation: Final = OVHCloudChatConfig() -lemonade_transformation: Final = LemonadeChatConfig() - -MOCK_RESPONSE_TYPE = str | Exception | dict | ModelResponse | ModelResponseStream - - -def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> ControlOptions: - control: Final = parse_control_options(kwargs) - match control: - case ControlOptions(): - return control - case InvalidControlOption(param=param, message=message): - if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True: - return ControlOptions() - raise litellm.BadRequestError(message=message, model=model, llm_provider=None, body={"param": param}) - case _: - return assert_never(control) - - -PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"}) -PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"}) - - -def _cost_map_pdf_input_support(model: str, custom_llm_provider: str) -> bool | None: - if custom_llm_provider in PROVIDERS_WITHOUT_DOCUMENT_INPUT or is_explicitly_disabled_factory( - model, custom_llm_provider, "supports_pdf_input" - ): - return False - if supports_pdf_input(model, custom_llm_provider): - return True - return None - - -def _deployment_pdf_input_support(model: str, base_model: str, custom_llm_provider: str) -> bool | None: - supports: Final = ( - _cost_map_pdf_input_support(model, custom_llm_provider), - _cost_map_pdf_input_support(base_model, custom_llm_provider), - ) - if False in supports: - return False - if True in supports: - return True - return None - - -def _declared_pdf_input_support( - model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None, base_model: str | None -) -> bool | None: - match model_info: - case {"supports_pdf_input": bool() as declared}: - return declared - case _ if base_model: - return _deployment_pdf_input_support(model, base_model, custom_llm_provider) - case _: - return _cost_map_pdf_input_support(model, custom_llm_provider) - - -def _pdf_input_unreadable( - model: str, - custom_llm_provider: str, - model_info: Mapping[str, object] | None, - base_model: str | None, - drop_params: bool, -) -> bool: - match _declared_pdf_input_support(model, custom_llm_provider, model_info, base_model): - case False: - return True - case None: - return drop_params and custom_llm_provider in PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT - case True: - return False - - -def _messages_without_unreadable_pdfs( - messages: Sequence[AllMessageValues], - model: str, - custom_llm_provider: str, - model_info: Mapping[str, object] | None, - base_model: str | None, - drop_params: bool, -) -> tuple[AllMessageValues, ...] | None: - if not messages_have_pdf_file_part(messages) or not _pdf_input_unreadable( - model, custom_llm_provider, model_info, base_model, drop_params - ): - return None - if not drop_params: - raise litellm.BadRequestError( - message=( - f"{custom_llm_provider}/{model} does not accept PDF input. " - "Remove the PDF, or set drop_params=True to send the request without it" - ), - model=model, - llm_provider=custom_llm_provider, - ) - readable_messages: Final = messages_without_pdf_file_parts(messages) - if not any(message["role"] == "user" for message in readable_messages): - raise litellm.BadRequestError( - message=( - f"{custom_llm_provider}/{model} does not accept PDF input, " - "and no user message is left once the PDF is dropped" - ), - model=model, - llm_provider=custom_llm_provider, - ) - return readable_messages - - -def _messages_readable_by_model( - messages: Sequence[AllMessageValues], - model: str, - custom_llm_provider: str, - model_info: Mapping[str, object] | None, - base_model: str | None, - drop_params: bool, - ensure_alternating_roles: bool | None, - user_continue_message: ChatCompletionUserMessage | None, - assistant_continue_message: ChatCompletionAssistantMessage | None, -) -> tuple[AllMessageValues, ...] | None: - pdf_free_messages: Final = _messages_without_unreadable_pdfs( - messages=messages, - model=model, - custom_llm_provider=custom_llm_provider, - model_info=model_info, - base_model=base_model, - drop_params=drop_params, - ) - if pdf_free_messages is None: - return None - return tuple( - get_completion_messages( - messages=list(pdf_free_messages), - ensure_alternating_roles=ensure_alternating_roles or False, - user_continue_message=user_continue_message, - assistant_continue_message=assistant_continue_message, - ) - ) ####### COMPLETION ENDPOINTS ################ @@ -706,7 +596,6 @@ async def acompletion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, ) - from litellm.types.llms.openai import AllMessageValues AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=kwargs, @@ -925,12 +814,6 @@ async def acompletion( ) -async def _resolve_dispatched_chat_response( - pending: Coroutine[object, object, ModelResponse | CustomStreamWrapper], -) -> ModelResponse | CustomStreamWrapper: - return await pending - - def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> ModelResponse: return ModelResponse(**cached_response_dict) @@ -961,4530 +844,18 @@ async def _async_streaming(response, model, custom_llm_provider, args): ) -def _handle_mock_potential_exceptions( - mock_response: str | Exception, - model: str, - custom_llm_provider: str | None = None, -): - if isinstance(mock_response, Exception): - if isinstance(mock_response, openai.APIError): - raise mock_response - raise litellm.MockException( - status_code=getattr(mock_response, "status_code", 500), - message=getattr(mock_response, "text", str(mock_response)), - llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), - model=model, - request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), - ) - elif isinstance(mock_response, str) and mock_response == "litellm.RateLimitError": - raise litellm.RateLimitError( - message="this is a mock rate limit error", - llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), - model=model, - ) - elif isinstance(mock_response, str) and mock_response == "litellm.ContextWindowExceededError": - raise litellm.ContextWindowExceededError( - message="this is a mock context window exceeded error", - llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), - model=model, - ) - elif isinstance(mock_response, str) and mock_response == "litellm.InternalServerError": - raise litellm.InternalServerError( - message="this is a mock internal server error", - llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), - model=model, - ) - elif isinstance(mock_response, str) and mock_response.startswith("Exception: content_filter_policy"): - raise litellm.MockException( - status_code=400, - message=mock_response, - llm_provider="azure", - model=model, - request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), - ) - - -def _handle_mock_timeout( - mock_timeout: bool | None, - timeout: float | str | httpx.Timeout | None, - model: str, -): - if mock_timeout is True and timeout is not None: - _sleep_for_timeout(timeout) - raise litellm.Timeout( - message="This is a mock timeout error", - llm_provider="openai", - model=model, - ) - - -async def _handle_mock_timeout_async( - mock_timeout: bool | None, - timeout: float | str | httpx.Timeout | None, - model: str, -): - if mock_timeout is True and timeout is not None: - await _sleep_for_timeout_async(timeout) - raise litellm.Timeout( - message="This is a mock timeout error", - llm_provider="openai", - model=model, - ) - - -def _sleep_for_timeout(timeout: float | str | httpx.Timeout): - if isinstance(timeout, float): - time.sleep(timeout) - elif isinstance(timeout, str): - time.sleep(float(timeout)) - elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None: - time.sleep(timeout.connect) - - -async def _sleep_for_timeout_async(timeout: float | str | httpx.Timeout): - if isinstance(timeout, float): - await asyncio.sleep(timeout) - elif isinstance(timeout, str): - await asyncio.sleep(float(timeout)) - elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None: - await asyncio.sleep(timeout.connect) - - -class _AdmissionReservation(LiteLLMBaseModel): - input_tokens: int | None = None - - -class _AdmissionMetadata(LiteLLMBaseModel): - user_api_key_budget_reservation: _AdmissionReservation | None = None - - -def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None: - reservations: Final = ( - _AdmissionMetadata.model_validate(kwargs.get(key) or {}).user_api_key_budget_reservation - for key in ("litellm_metadata", "metadata") - ) - return next( - ( - reservation.input_tokens - for reservation in reservations - if reservation and reservation.input_tokens is not None - ), - None, - ) - - -def admitted_prompt_token_counter(prompt_tokens: int | None) -> Callable[[], int] | None: - if prompt_tokens is None: - return None - return lambda: prompt_tokens - - -def mock_completion( - model: str, - messages: list, - stream: bool | None = False, - n: int | None = None, - mock_response: MOCK_RESPONSE_TYPE | None = "This is a mock request", - mock_tool_calls: list | None = None, - mock_timeout: bool | None = False, - logging=None, - custom_llm_provider=None, - timeout: float | str | httpx.Timeout | None = None, - prompt_tokens: int | None = None, - **kwargs, -): - """ - Generate a mock completion response for testing or debugging purposes. - - This is a helper function that simulates the response structure of the OpenAI completion API. - - Parameters: - model (str): The name of the language model for which the mock response is generated. - messages (List): A list of message objects representing the conversation context. - stream (bool, optional): If True, returns a mock streaming response (default is False). - mock_response (str, optional): The content of the mock response (default is "This is a mock request"). - mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False). - timeout (float, optional): The timeout value to use for the mock response (default is None). - **kwargs: Additional keyword arguments that can be used but are not required. - - Returns: - litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response. - - Raises: - Exception: If an error occurs during the generation of the mock completion response. - Note: - - This function is intended for testing or debugging purposes to generate mock completion responses. - - If 'stream' is True, it returns a response that mimics the behavior of a streaming completion. - """ - try: - is_acompletion: Final = kwargs.get("acompletion") or False - if mock_response is None: - mock_response = "This is a mock request" - - _handle_mock_timeout(mock_timeout=mock_timeout, timeout=timeout, model=model) - - ## LOGGING - if logging is not None: - logging.pre_call( - input=messages, - api_key="mock-key", - ) - - if isinstance(mock_response, str) or isinstance(mock_response, Exception): - _handle_mock_potential_exceptions( - mock_response=mock_response, - model=model, - custom_llm_provider=custom_llm_provider, - ) - - mock_response = cast( - str | dict | ModelResponse | ModelResponseStream, mock_response - ) # after this point, mock_response is a string, dict, ModelResponse, or ModelResponseStream - if isinstance(mock_response, str) and mock_response.startswith("Exception: mock_streaming_error"): - mock_response = litellm.MockException( - message="This is a mock error raised mid-stream", - llm_provider="anthropic", - model=model, - status_code=529, - ) - time_delay: Final = kwargs.get("mock_delay", None) - if time_delay is not None and not is_acompletion: - time.sleep(time_delay) - - if isinstance(mock_response, dict): - return ModelResponse(**mock_response) - - if isinstance(mock_response, ModelResponse): - if not stream: - return mock_response - # convert to ModelResponseStream - mock_response = convert_model_response_to_streaming(mock_response) - - model_response: ModelResponse | ModelResponseStream = ModelResponse() - - if stream is True: - model_response = ModelResponseStream() - count_prompt_tokens: Final = admitted_prompt_token_counter(prompt_tokens) - # don't try to access stream object, - if kwargs.get("acompletion", False) is True: - return CustomStreamWrapper( - completion_stream=async_mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens - ), - model=model, - custom_llm_provider="openai", - logging_obj=logging, - count_prompt_tokens=count_prompt_tokens, - ) - return CustomStreamWrapper( - completion_stream=mock_completion_streaming_obj( - model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens - ), - model=model, - custom_llm_provider="openai", - logging_obj=logging, - count_prompt_tokens=count_prompt_tokens, - ) - if isinstance(mock_response, litellm.MockException): - raise mock_response - # At this point, mock_response must be a string (all other types have been handled or returned early) - mock_response = cast(str, mock_response) - - if n is None: - model_response.choices[0].message.content = mock_response - else: - _all_choices: Final = [] - for i in range(n): - _choice = litellm.utils.Choices( - index=i, - message=litellm.utils.Message(content=mock_response, role="assistant"), - ) - _all_choices.append(_choice) - model_response.choices = _all_choices - model_response.created = int(time.time()) - model_response.model = model - - if mock_tool_calls: - model_response.choices[0].message.tool_calls = [ - ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls - ] - - usage_prompt_tokens: Final = ( - prompt_tokens if prompt_tokens is not None else DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT - ) - setattr( - model_response, - "usage", - Usage( - prompt_tokens=usage_prompt_tokens, - completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - total_tokens=usage_prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - ), - ) - - if custom_llm_provider is not None: - model_response.hidden_params["custom_llm_provider"] = custom_llm_provider - else: - try: - _, inferred_provider, _, _ = litellm.utils.get_llm_provider(model=model) - model_response.hidden_params["custom_llm_provider"] = inferred_provider - except Exception: - # dont let setting a hidden param block a mock_respose - pass - - if logging is not None: - logging.post_call( - input=messages, - api_key="my-secret-key", - original_response="my-original-response", - ) - - return model_response - - except Exception as e: - if isinstance(e, openai.APIError): - raise e - raise Exception(f"Mock completion response failed - {e}") - - -from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site - _resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper -) -from litellm.responses.bridge_check import ( - bridge_reasoning_effort, - keep_registered_mode, - str_keyed_mapping, -) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site responses_api_bridge_check as responses_api_bridge_check, # pyright: ignore[reportUnknownVariableType] # noqa: PLC0414 # re-export of moved function ) -_ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) - - -def _history_without_unanswered_tool_search_calls( - messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig -) -> tuple[object, ...] | None: - from litellm.responses.litellm_completion_transformation.tool_search import ( - chat_messages_without_tool_search_calls, - chat_route_rejects_unanswered_tool_calls, - unanswered_tool_search_call_ids, - ) - - if not chat_route_rejects_unanswered_tool_calls(custom_llm_provider, provider_config): - return None - unanswered_search_call_ids: Final = unanswered_tool_search_call_ids(messages) - if not unanswered_search_call_ids: - return None - return chat_messages_without_tool_search_calls(messages, unanswered_search_call_ids) - - -def _without_anthropic_only_tool_keys(tool: dict[str, object]) -> dict[str, object]: - kept: Final = {key: value for key, value in tool.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS} - function: Final = tool.get("function") - if not isinstance(function, dict): - return kept - return { - **kept, - "function": {key: value for key, value in function.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS}, - } - - -def _drop_anthropic_only_tool_keys(tools: list[dict[str, object]] | None) -> list[dict[str, object]] | None: - if tools is None: - return None - return [_without_anthropic_only_tool_keys(tool) if isinstance(tool, dict) else tool for tool in tools] - - -def _azure_completion_fakes_stream(model: str, base_model: str | None) -> bool: - if not litellm.AzureOpenAIO1Config().is_o_series_model(model=base_model or model): - return False - o_series_handler_config: Final = ProviderConfigManager.get_provider_chat_config( - model=model, provider=LlmProviders.AZURE - ) - return o_series_handler_config is not None and o_series_handler_config.should_fake_stream( - model=model, stream=True, custom_llm_provider=LlmProviders.AZURE.value - ) - - -def _completion_fakes_stream( - model: str, - custom_llm_provider: str | None, - provider_config: BaseConfig | None, - optional_params: Mapping[str, object], - *, - base_model: str | None, - bridges_to_responses_api: bool, -) -> bool: - if optional_params.get("fake_stream") is True: - return True - if custom_llm_provider is None: - return False - if bridges_to_responses_api: - responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( - provider=custom_llm_provider, model=model - ) - return responses_config is not None and responses_config.should_fake_stream( - model=model, stream=True, custom_llm_provider=custom_llm_provider - ) - if ( - custom_llm_provider == LlmProviders.VERTEX_AI.value - and get_vertex_ai_model_route(model=model, litellm_params={"base_model": base_model}) - == VertexAIModelRoute.GEMMA - ): - return True - if isinstance(provider_config, litellm.AmazonAI21Config): - return True - if custom_llm_provider == LlmProviders.AZURE.value: - return _azure_completion_fakes_stream(model, base_model) - return provider_config is not None and provider_config.should_fake_stream( - model=model, stream=True, custom_llm_provider=custom_llm_provider - ) - - -class _ProxyAuthHeadersProvider(Protocol): - def get_auth_headers(self) -> Mapping[str, str]: ... - - -def _proxy_auth_headers(proxy_auth: _ProxyAuthHeadersProvider) -> Mapping[str, str]: - return proxy_auth.get_auth_headers() - - -def _provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]: - return config.items() - def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values -def _build_custom_pricing_entry( - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None = None, -) -> dict: - """Build a complete model cost entry from kwargs and model_info. - - Collects all CustomPricingLiteLLMParams fields present in kwargs and - merges metadata from model_info (mode, supports_prompt_caching, max_tokens) - so that register_model() receives the full pricing configuration. - """ - entry: Final[dict] = {"litellm_provider": custom_llm_provider} - - for field_name in CustomPricingLiteLLMParams.model_fields: - value = kwargs.get(field_name) - if value is not None: - entry[field_name] = value - - if model_info and isinstance(model_info, dict): - for key in ("mode", "supports_prompt_caching", "max_tokens"): - if key in model_info and model_info[key] is not None: - entry.setdefault(key, model_info[key]) - - return entry - - -def _get_router_deployment_id(kwargs: dict) -> str | None: - for metadata_key in ("litellm_metadata", "metadata"): - metadata = kwargs.get(metadata_key) or {} - if not isinstance(metadata, dict): - continue - deployment_model_info = metadata.get("model_info") or {} - if not isinstance(deployment_model_info, dict): - continue - deployment_id = deployment_model_info.get("id") - if deployment_id is not None: - return str(deployment_id) - return None - - -def _record_agentic_loop_params( - logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool -) -> None: - if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): - return - logging_obj.record_agentic_loop_params( - AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) - ) - - -def _register_custom_pricing_for_request( - model: str, - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None, -) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) - return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) - ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - - -def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: - return ctx.metadata - - -def _dispatch_client_http(ctx: CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None: - return ctx.client - - -def _dispatch_client_azure( - ctx: CompletionDispatchContext, -) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None: - return ctx.client - - -def _dispatch_client_openai(ctx: CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None: - return ctx.client - - -def _complete_azure(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - _azure_detection_model: Final = ctx._azure_detection_model - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - api_version = ctx.api_version - client: Final = _dispatch_client_azure(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - max_retries: Final = ctx.max_retries - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - dynamic_params = False - if client is not None and (isinstance(client, openai.AzureOpenAI) or isinstance(client, openai.AsyncAzureOpenAI)): - dynamic_params = check_dynamic_azure_params( - azure_client_params={"api_version": api_version}, - azure_client=client, - ) - - api_type: Final = get_secret("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") - - api_version = ( - api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION - ) - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) - - azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( - "AZURE_AD_TOKEN" - ) - - azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) - azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - if max_retries is not None: - optional_params["max_retries"] = max_retries - - if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIO1Config.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = azure_o1_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - custom_llm_provider=custom_llm_provider, - ) - else: - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response = azure_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - api_type=api_type, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_azure_text(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - api_version = ctx.api_version - client: Final = _dispatch_client_azure(ctx) - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - api_type: Final = get_secret_str("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - - if api_base is None: - raise ValueError( - "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." - ) - - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) - - azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( - "AZURE_AD_TOKEN" - ) - - azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) - azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - ## LOAD CONFIG - if set - config: Final = litellm.AzureOpenAIConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response: Final = azure_text_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=cast(str, api_version), - api_type=api_type, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) - - return response - - -def _complete_deepseek(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_azure_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo - - azure_ai_route: Final = AzureFoundryModelInfo.get_azure_ai_route(model) - - # Check if this is an agents route - model format: azure_ai/agents/ - if azure_ai_route == "agents": - from litellm.llms.azure_ai.agents import AzureAIAgentsConfig - - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - response = AzureAIAgentsConfig.completion( - model=model, - messages=messages, - api_base=api_base, - api_key=api_key, - model_response=model_response, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - acompletion=acompletion, - stream=stream, - headers=headers or litellm.headers, - ) - - # Check if this is a Claude model - route to Azure Anthropic handler - elif "claude" in model.lower(): - # Use Azure Anthropic handler for Claude models - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - if api_base: - api_base = azure_anthropic_messages_url(api_base) - - response = azure_anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - response = response - else: - # Non-Claude models use standard Azure AI flow - api_base = AzureFoundryModelInfo.get_api_base(api_base) - # set API KEY - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - - ## FOR COHERE - if "command-r" in model: # make sure tool call in messages are str - messages = stringify_json_tool_call_content(messages=messages) - - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, # pass AsyncOpenAI, OpenAI client - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) - - return response - - -def _complete_text_completion_openai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_openai(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - text_completion: Final = ctx.text_completion - timeout: Final = ctx.timeout - - openai.api_type = "openai" - - api_base = ( - api_base - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - - openai.api_version = None - # set API KEY - - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") - - headers = headers or litellm.headers - - ## LOAD CONFIG - if set - config: Final = litellm.OpenAITextCompletionConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - if litellm.organization: - openai.organization = litellm.organization - - ## COMPLETION CALL - _response = openai_text_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - acompletion=acompletion, - client=client, # pass AsyncOpenAI, OpenAI client - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - ) - - if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: - # convert to chat completion response - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_fireworks_ai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - optional_params: Final = ( - provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) - if isinstance(provider_config, litellm.FireworksAIConfig) - else ctx.optional_params - ) - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_together_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args=MappingProxyType({"headers": headers}), - ) - raise - - return response - - -def _complete_heroku(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_ragflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_scaledown(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - try: - return base_llm_http_handler.completion( - model=ctx.model, - messages=ctx.messages, - headers=ctx.headers, - model_response=ctx.model_response, - api_key=ctx.api_key, - api_base=ctx.api_base, - acompletion=ctx.acompletion, - logging_obj=ctx.logging, - optional_params=ctx.optional_params, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value - float | httpx.Timeout, ctx.timeout - ), - client=_dispatch_client_http(ctx), - custom_llm_provider=ctx.custom_llm_provider, - encoding=_get_encoding(), - stream=ctx.stream, - provider_config=ctx.provider_config, - ) - except Exception as exc: - ctx.logging.post_call( - input=ctx.messages, - api_key=ctx.api_key, - original_response=str(exc), - additional_args={"headers": ctx.headers}, - ) - raise - - -def _complete_xai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_groq(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("GROQ_API_BASE") - or "https://api.groq.com/openai/v1" - ) - - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.groq_key - or get_secret("GROQ_API_KEY") - ) - - headers = headers or litellm.headers - - ## LOAD CONFIG - if set - config: Final = litellm.GroqChatConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_opencode_go(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - return base_llm_http_handler.completion( # pyright: ignore[reportUnknownMemberType] # shared legacy handler has untyped request mappings - model=ctx.model, - stream=ctx.stream, - messages=ctx.messages, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # existing dispatch context carries validated messages - acompletion=ctx.acompletion, - api_base=ctx.api_base, - model_response=ctx.model_response, - optional_params=ctx.optional_params, # pyright: ignore[reportUnknownMemberType] # existing dispatch context carries the mapped provider parameters - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - custom_llm_provider=ctx.custom_llm_provider, - timeout=ctx.timeout, # pyright: ignore[reportArgumentType] # completion normalizes timeout before constructing this context - headers=ctx.headers, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy context retains caller headers - encoding=_get_encoding(), - api_key=ctx.api_key, - logging_obj=ctx.logging, - client=_dispatch_client_http(ctx), - ) - - -def _complete_bedrock_mantle( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") - api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") - headers = headers or litellm.headers - config: Final = bedrock_mantle_chat_config(model).get_config() - for k, v in _provider_config_items(config): - if k not in optional_params: - optional_params[k] = v - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - -def _complete_a2a(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - ( - api_base, - api_key, - headers, - ) = litellm.A2AConfig.resolve_agent_config_from_registry( - agent_name=model, - api_base=api_base, - api_key=api_key, - headers=headers, - optional_params=optional_params, - ) - - # Fall back to environment variables and defaults - api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") - - if api_base is None: - raise Exception( - "api_base is required for A2A provider. " - "Either provide api_base parameter, set A2A_API_BASE environment variable, " - "or register the agent in the proxy with model='a2a/'." - ) - - headers = headers or litellm.headers - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - - -def _complete_gigachat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = ( - api_key - or litellm.api_key - or litellm.gigachat_key - or get_secret("GIGACHAT_API_KEY") - or get_secret("GIGACHAT_CREDENTIALS") - ) - - headers = headers or litellm.headers or {} - - ## COMPLETION CALL - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_sap(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - headers = headers or litellm.headers - ## LOAD CONFIG - if set - config: Final = litellm.GenAIHubOrchestrationConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - return sap_gen_ai_hub_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - shared_session=shared_session, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - api_key=api_key, - api_base=api_base, - stream=stream, - ) - - -def _complete_aiohttp_openai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - if http2_enabled(): - verbose_logger.warning( - "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1" - ) - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - return base_llm_aiohttp_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - -def _complete_cometapi(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.cometapi_key or get_secret_str("COMETAPI_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret_str("COMETAPI_API_BASE") or "https://api.cometapi.com/v1" - - ## COMPLETION CALL - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - ## LOGGING - logging.post_call(input=messages, api_key=api_key, original_response=response) - - return response - - -def _complete_minimax(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/v1" - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call(input=messages, api_key=api_key, original_response=response) - - return response - - -def _complete_hosted_vllm(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call(input=messages, api_key=api_key, original_response=response) - - return response - - -def _complete_custom_openai( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - extra_headers = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - metadata: Final = _dispatch_metadata(ctx) - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - organization = ctx.organization - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - api_base = _resolve_openai_api_base(api_base) - if custom_llm_provider == "bigmodel" and not api_key: - raise litellm.AuthenticationError( - message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model - ) - organization = ( - organization - or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION")) - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 - ) - openai.organization = organization - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - - headers = headers or litellm.headers - outbound_headers: Final = ( - headers - if provider_config is None - else with_attribution_headers(provider_config.get_attribution_headers(), headers) - ) - - # Add GitHub Copilot headers (same as /responses endpoint does) - if custom_llm_provider == "github_copilot": - from litellm.llms.github_copilot.authenticator import Authenticator - from litellm.llms.github_copilot.common_utils import ( - get_copilot_default_headers, - pin_session_authorization, - ) - from litellm.llms.github_copilot.per_user_auth import ( - require_github_copilot_user_session, - ) - - user_session: Final = require_github_copilot_user_session(litellm_params) - copilot_headers: Final = get_copilot_default_headers( - user_session.token if user_session is not None else Authenticator().get_api_key() - ) - if extra_headers: - copilot_headers.update( - cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict - ) - if user_session is not None: - pin_session_authorization(copilot_headers, user_session.token) - extra_headers = copilot_headers - - use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER") - - if extra_headers is not None and not use_base_llm_http_handler: - optional_params["extra_headers"] = extra_headers - - if litellm.enable_preview_features and metadata is not None: # [PREVIEW] allow metadata to be passed to OPENAI - openai_metadata: Final = get_requester_metadata(metadata) - if openai_metadata is not None: - optional_params["metadata"] = openai_metadata - - ## LOAD CONFIG - if set - config: Final = litellm.OpenAIConfig.get_config() - for k, v in _provider_config_items(config): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - try: - if use_base_llm_http_handler: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=outbound_headers, - client=client, - provider_config=provider_config, - ) - else: - response = openai_chat_completions.completion( - model=model, - messages=messages, - headers=outbound_headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - organization=organization, - custom_llm_provider=custom_llm_provider, - shared_session=shared_session, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": outbound_headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": outbound_headers}, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_mistral(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") - api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1" - - return base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - - -def _complete_replicate(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - replicate_key: Final = ( - api_key - or litellm.replicate_key - or litellm.api_key - or get_secret("REPLICATE_API_KEY") - or get_secret("REPLICATE_API_TOKEN") - ) - - api_base = api_base or litellm.api_base or get_secret("REPLICATE_API_BASE") or "https://api.replicate.com/v1" - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - - model_response = replicate_chat_completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=replicate_key, - logging_obj=logging, - custom_prompt_dict=custom_prompt_dict, - acompletion=acompletion, - headers=headers, - ) - - if optional_params.get("stream", False) is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=replicate_key, - original_response=model_response, - ) - - return model_response - - -def _complete_anthropic_text( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - api_base = cast( - str | None, - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/complete", - ) - - # Check if we should disable automatic URL suffix appending - disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if api_base is not None and not disable_url_suffix: - api_base = ensure_api_route(api_base, "v1/complete") - elif disable_url_suffix: - verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix") - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="anthropic_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - -def _complete_anthropic(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - # call /messages - # default route for all anthropic models - api_base = cast( - str | None, - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/messages", - ) - - # Check if we should disable automatic URL suffix appending - disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if api_base is not None and not disable_url_suffix: - api_base = ensure_api_route(api_base, "v1/messages") - elif disable_url_suffix: - verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix") - - response: Final = anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - return response - - -def _complete_nlp_cloud(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - - nlp_cloud_key: Final = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret("NLP_CLOUD_API_BASE") or "https://api.nlpcloud.io/v1/gpu/" - - response = nlp_cloud_chat_completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=nlp_cloud_key, - logging_obj=logging, - ) - - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - response, - model, - custom_llm_provider="nlp_cloud", - logging_obj=logging, - ) - - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_aleph_alpha(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - api_key: Final = ctx.api_key - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - aleph_alpha_key: Final = ( - api_key - or litellm.aleph_alpha_key - or get_secret("ALEPH_ALPHA_API_KEY") - or get_secret("ALEPHALPHA_API_KEY") - or litellm.api_key - ) - - api_base = ( - api_base or litellm.api_base or get_secret("ALEPH_ALPHA_API_BASE") or "https://api.aleph-alpha.com/complete" - ) - - model_response = aleph_alpha.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - default_max_tokens_to_sample=litellm.max_tokens, - api_key=aleph_alpha_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="aleph_alpha", - logging_obj=logging, - ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_cohere_chat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - extra_headers: Final = ctx.extra_headers - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - cohere_key: Final = ( - api_key - or litellm.cohere_key - or get_secret_str("COHERE_API_KEY") - or get_secret_str("CO_API_KEY") - or litellm.api_key - ) - - cohere_route: Final = CohereModelInfo.get_cohere_route(model) - verbose_logger.debug("Cohere route: %s", cohere_route) - # Set API base based on route - if cohere_route == "v2": - api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat" - # Remove v2/ prefix from model name for the actual API call - if "v2/" in model: - model = model.replace("v2/", "") - else: - api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.ai/v1/chat" - - headers = headers or litellm.headers or {} - if headers is None: - headers = {} - - if extra_headers is not None: - headers.update(extra_headers) - - verbose_logger.debug("Model: %s, API Base: %s", model, api_base) - verbose_logger.debug("Provider Config: %s", provider_config) - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cohere_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=cohere_key, - provider_config=provider_config, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - -def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - api_key: Final = ctx.api_key - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - - maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key - - api_base = api_base or litellm.api_base or get_secret("MARITALK_API_BASE") or "https://chat.maritaca.ai/api" - - return openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=maritalk_key, - logging_obj=logging, - custom_llm_provider="maritalk", - custom_prompt_dict=custom_prompt_dict, - ) - - -def _complete_amazon_nova(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - api_key = ctx.api_key - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - api_key = api_key or litellm.amazon_nova_api_key or get_secret_str("AMAZON_NOVA_API_KEY") or litellm.api_key - api_base = ( - api_base or litellm.api_base or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1" - ) - return openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - custom_prompt_dict=custom_prompt_dict, - ) - - -def _complete_huggingface(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - huggingface_key: Final = ( - api_key - or litellm.huggingface_key - or os.environ.get("HF_TOKEN") - or os.environ.get("HUGGINGFACE_API_KEY") - or litellm.api_key - ) - hf_headers: Final = headers or litellm.headers - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=hf_headers, - model_response=model_response, - api_key=huggingface_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - -def _complete_oci(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - -def _complete_compactifai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key - - api_base = api_base or "https://api.compactif.ai/v1" - - ## COMPLETION CALL - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - -def _complete_oobabooga(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = ctx.api_base - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - model_response = oobabooga.completion( - model=model, - messages=messages, - model_response=model_response, - api_base=api_base, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - api_key=None, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="oobabooga", - logging_obj=logging, - ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_databricks(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = DatabricksBase.resolve_api_base(ctx.api_base) - api_key: Final = DatabricksBase.resolve_api_key(ctx.api_key) - client: Final = _dispatch_client_http(ctx) - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - headers = headers or litellm.headers - - ## COMPLETION CALL - try: - response: Final = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="databricks", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) - - return response - - -def _complete_datarobot(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config: Final = ctx.provider_config - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - - -def _complete_openrouter(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - - api_key = ( - api_key - or litellm.api_key - or litellm.openrouter_key - or get_secret_str("OPENROUTER_API_KEY") - or get_secret_str("OR_API_KEY") - ) - - openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM" - - openrouter_headers: Final = { - "HTTP-Referer": openrouter_site_url, - "X-Title": openrouter_app_name, - } - - _headers: Final = headers or litellm.headers - if _headers: - openrouter_headers.update(_headers) - - headers = openrouter_headers - - ## Load Config - config: Final = litellm.OpenrouterConfig.get_config() - for k, v in _provider_config_items(config): - if k == "extra_body": - # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - - ## COMPLETION CALL - response: Final = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="openrouter", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call(input=messages, api_key=openai.api_key, original_response=response) - - return response - - -def _complete_nadir(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE - api_key: Final = ctx.api_key - - response: Final = base_llm_http_handler.completion( - model=ctx.model, - stream=ctx.stream, - messages=ctx.messages, - acompletion=ctx.acompletion, - api_base=api_base, - model_response=ctx.model_response, - optional_params=ctx.optional_params, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - custom_llm_provider="nadir", - timeout=ctx.timeout, - headers=ctx.headers or litellm.headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=ctx.logging, - client=_dispatch_client_http(ctx), - ) - ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) - - return response - - -def _complete_vercel_ai_gateway( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") - or "https://ai-gateway.vercel.sh/v1" - ) - - api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") - - vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" - vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM" - - vercel_headers: Final = { - "http-referer": vercel_site_url, - "x-title": vercel_app_name, - } - - _headers: Final = headers or litellm.headers - if _headers: - vercel_headers.update(_headers) - - headers = vercel_headers - - ## Load Config - config: Final = litellm.VercelAIGatewayConfig.get_config() - for k, v in _provider_config_items(config): - if k == "extra_body": - # we use openai 'extra_body' to pass vercel specific params - providerOptions - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - - ## COMPLETION CALL - response: Final = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="vercel_ai_gateway", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call(input=messages, api_key=openai.api_key, original_response=response) - - return response - - -def _complete_edenai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = litellm.EdenAIChatConfig.get_api_base(ctx.api_base) - api_key: Final = litellm.EdenAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) - response: Final = base_llm_http_handler.completion( - model=ctx.model, - messages=ctx.messages, - api_base=api_base, - custom_llm_provider="edenai", - model_response=ctx.model_response, - encoding=_get_encoding(), - logging_obj=ctx.logging, - optional_params=ctx.optional_params, - timeout=ctx.timeout, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - acompletion=ctx.acompletion, - stream=ctx.stream, - api_key=api_key, - headers=ctx.headers or litellm.headers, - client=_dispatch_client_http(ctx), - provider_config=ctx.provider_config, - ) - ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) - return response - - -def _complete_fal_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - if ctx.stream: - raise litellm.FalAIError( - status_code=400, - message="fal_ai chat completions do not support streaming", - ) - api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base) - api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) - response: Final = base_llm_http_handler.completion( - model=ctx.model, - messages=ctx.messages, - api_base=api_base, - custom_llm_provider="fal_ai", - model_response=ctx.model_response, - encoding=_get_encoding(), - logging_obj=ctx.logging, - optional_params=ctx.optional_params, - timeout=ctx.timeout, - litellm_params=ctx.litellm_params, - shared_session=ctx.shared_session, - acompletion=ctx.acompletion, - stream=ctx.stream, - api_key=api_key, - headers=ctx.headers or litellm.headers, - client=_dispatch_client_http(ctx), - provider_config=ctx.provider_config, - ) - ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) - return response - - -def _complete_vertex_ai_beta( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - vertex_ai_project: Final = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location: Final = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials: Final = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) - - gemini_api_key: Final = ( - api_key - or get_api_key_from_env() - or get_secret("PALM_API_KEY") # older palm api key should also work - or litellm.api_key - ) - - api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") - new_params: Final = safe_deep_copy(optional_params or {}) - return vertex_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=gemini_api_key, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - client=client, - api_base=api_base, - extra_headers=headers, - ) - - -def _complete_vertex_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - vertex_ai_project: Final = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location: Final = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials: Final = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) - - api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") - - new_params: Final = safe_deep_copy(optional_params or {}) - model_route: Final = get_vertex_ai_model_route(model=model, litellm_params=litellm_params) - - if model_route == VertexAIModelRoute.PARTNER_MODELS: - model_response = vertex_partner_models_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.GEMINI: - model_response = vertex_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=None, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - client=client, - api_base=api_base, - extra_headers=headers, - ) - elif model_route == VertexAIModelRoute.GEMMA: - # Vertex Gemma Models with custom prediction endpoint - model_response = vertex_gemma_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.MODEL_GARDEN: - # Vertex Model Garden - OpenAI compatible models - model_response = vertex_model_garden_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.AGENT_ENGINE: - # Vertex AI Agent Engine (Reasoning Engines) - from litellm.llms.vertex_ai.agent_engine.transformation import ( - VertexAgentEngineConfig, - ) - - vertex_agent_engine_config: Final = VertexAgentEngineConfig() - - # Update litellm_params with vertex credentials - litellm_params["vertex_project"] = vertex_ai_project - litellm_params["vertex_location"] = vertex_ai_location - litellm_params["vertex_credentials"] = vertex_credentials - - model_response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - model_response=model_response, - optional_params=new_params, - litellm_params=litellm_params, - encoding=_get_encoding(), - api_key=None, - api_base=api_base, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, - custom_llm_provider="vertex_ai", - provider_config=vertex_agent_engine_config, - headers=headers or {}, - ) - else: # VertexAIModelRoute.NON_GEMINI - model_response = vertex_ai_non_gemini.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - ) - - if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vertex_ai", - logging_obj=logging, - ) - return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_predibase(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - tenant_id: Final = ( - optional_params.pop("tenant_id", None) - or optional_params.pop("predibase_tenant_id", None) - or litellm.predibase_tenant_id - or get_secret("PREDIBASE_TENANT_ID") - ) - - if tenant_id is None: - raise ValueError( - "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." - ) - - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or get_secret("PREDIBASE_API_BASE") - ) - - api_key = api_key or litellm.api_key or litellm.predibase_key or get_secret("PREDIBASE_API_KEY") - - _model_response: Final = predibase_chat_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - tenant_id=tenant_id, - timeout=timeout, - ) - - if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return _model_response - return _model_response - - -def _complete_text_completion_codestral( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict: Final = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or "https://codestral.mistral.ai/v1/fim/completions" - ) - - api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") - - text_completion_model_response: Final = litellm.TextCompletionResponse(stream=stream) - - _model_response: Final = codestral_text_completions.completion( - model=model, - messages=messages, - model_response=text_completion_model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - timeout=timeout, - ) - - if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_text_completion_inception( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_openai(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - text_completion: Final = ctx.text_completion - timeout: Final = ctx.timeout - - passed_api_base: Final = api_base or optional_params.pop("api_base", None) or optional_params.pop("base_url", None) - api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" - # FIM is served at `/v1/fim/completions`; the OpenAI client appends - # `/completions`, so point it at the `/v1/fim` base. - api_base = ensure_api_route(api_base, "fim") - - # Don't forward the server-managed Inception key to a caller-supplied - # api_base; only resolve it for the default/server base, or when the - # caller passes their own key. - if passed_api_base is None or api_key: - api_key = api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") - - _response = openai_text_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - custom_llm_provider="text-completion-inception", - api_base=api_base, - acompletion=acompletion, - client=client, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - ) - - if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - - if optional_params.get("stream", False) or acompletion is True: - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_sagemaker_chat( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - custom_prompt_dict: Final = ctx.custom_prompt_dict - hf_model_name: Final = ctx.hf_model_name - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - - return sagemaker_llm.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - custom_prompt_dict=custom_prompt_dict, - hf_model_name=hf_model_name, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - ) - - -_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) -_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) -_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str]) -_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) -_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) - - -_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - "aws_region_name", -) - - -def _native_output_fallback( - ctx: CompletionDispatchContext, - first_attempt_params: Mapping[str, object], - first_attempt_litellm_params: Mapping[str, object], -) -> NativeOutputFallback: - litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) - remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) - return NativeOutputFallback( - optional_params={ - **{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"}, - **{ - key: first_attempt_params[key] - for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS - if key in first_attempt_params - }, - }, - litellm_params=litellm_params, - ) - - -def _converse_native_output_fallback( - ctx: CompletionDispatchContext, - first_attempt_params: Mapping[str, object], - first_attempt_litellm_params: Mapping[str, object], -) -> NativeOutputFallback | None: - fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params) - return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback - - -def _native_output_attempt_timeout( - ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None -) -> float | str | httpx.Timeout | None: - if fallback is None: - return ctx.timeout - retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params) - ctx.logging.record_applied_timeout(retry_timeout) - return retry_timeout - - -def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - provider_config = ctx.provider_config - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - - if "aws_bedrock_client" in optional_params: - verbose_logger.warning( - "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." - ) - # Extract credentials for legacy boto3 client and pass thru to httpx - aws_bedrock_client: Final = optional_params.pop("aws_bedrock_client") - creds: Final = aws_bedrock_client._get_credentials().get_frozen_credentials() - - if creds.access_key: - optional_params["aws_access_key_id"] = creds.access_key - if creds.secret_key: - optional_params["aws_secret_access_key"] = creds.secret_key - if creds.token: - optional_params["aws_session_token"] = creds.token - if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None: - optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name - - additional_drop_params: Final = ( - _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"]) - if ctx.kwargs.get("additional_drop_params") is not None - else None - ) - bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params) - if bedrock_route == "claude_platform": - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=LlmProviders.BEDROCK, - ) - model = BedrockModelInfo.get_claude_platform_model(model) - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - elif bedrock_route == "converse": - model = without_bedrock_route_prefix(model) - converse_model: Final = model - - def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: - return bedrock_converse_chat_completion.completion( - model=converse_model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - optional_params=optional_params if fallback is None else dict(fallback.optional_params), - litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - extra_headers=headers, # Use merged headers instead of original extra_headers - timeout=_native_output_attempt_timeout(ctx, fallback), - acompletion=acompletion, - client=client, - api_base=api_base, - api_key=api_key, - ) - - converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) - response = call_with_native_output_fallback( - converse_attempt, - ( - partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) - else None - ), - converse_model, - strict_tools_fallback=( - partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params)) - if drops_unsupported_params(litellm_params.get("drop_params")) - else None - ), - ) - elif bedrock_route == "converse_like": - model = model.replace("converse_like/", "") - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - else: - - def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params if fallback is None else dict(fallback.optional_params), - litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), - custom_llm_provider="bedrock", - timeout=_native_output_attempt_timeout(ctx, fallback), - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) - response = call_with_native_output_fallback( - invoke_attempt, - ( - partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) - if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) - else None - ), - model, - ) - - return response - - -def _complete_watsonx(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_prompt_dict: Final = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - timeout: Final = ctx.timeout - - return watsonx_chat_completion.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - custom_llm_provider="watsonx", - ) - - -def _complete_watsonx_text( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = ( - api_key - or optional_params.pop("apikey", None) - or get_secret_str("WATSONX_APIKEY") - or get_secret_str("WATSONX_API_KEY") - or get_secret_str("WX_API_KEY") - ) - - api_base = ( - api_base - or optional_params.pop( - "url", - optional_params.pop("api_base", optional_params.pop("base_url", None)), - ) - or get_secret_str("WATSONX_API_BASE") - or get_secret_str("WATSONX_URL") - or get_secret_str("WX_URL") - or get_secret_str("WML_URL") - ) - - wx_credentials: Final = optional_params.pop( - "wx_credentials", - optional_params.pop("watsonx_credentials", None), # follow {provider}_credentials, same as vertex ai - ) - - token: str | None = None - if wx_credentials is not None: - api_base = wx_credentials.get("url", api_base) - api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key)) - token = wx_credentials.get( - "token", - wx_credentials.get( - "watsonx_token", None - ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' - ) - - if token is not None: - optional_params["token"] = token - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="watsonx_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_vllm(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - custom_prompt_dict = ctx.custom_prompt_dict - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - model_response = vllm_handler.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - - if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] - # don't try to access stream object, - return CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vllm", - logging_obj=logging, - ) - - ## RESPONSE OBJECT - return model_response - - -def _complete_ollama(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_ollama_chat(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" - - api_key = api_key or litellm.ollama_key or os.environ.get("OLLAMA_API_KEY") or litellm.api_key - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - - -def _complete_triton(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) - - -def _complete_cloudflare(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - custom_prompt_dict = ctx.custom_prompt_dict - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.cloudflare_api_key or litellm.api_key or get_secret("CLOUDFLARE_API_KEY") - api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE") - - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cloudflare", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - - -def _complete_petals(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base = ctx.api_base - client: Final = _dispatch_client_http(ctx) - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response = ctx.model_response - optional_params: Final = ctx.optional_params - stream = ctx.stream - - api_base = api_base or litellm.api_base - - stream = optional_params.pop("stream", False) - model_response = petals_handler.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - client=client, - ) - if stream is True: ## [BETA] - # Fake streaming for petals - resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"]) - return CustomStreamWrapper( - resp_string, - model, - custom_llm_provider="petals", - logging_obj=logging, - ) - return model_response - - -def _complete_snowflake(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - injected_client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - client: Final = ( - injected_client if injected_client is not None else (HTTPHandler(timeout=timeout) if stream is False else None) - ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible - - try: - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - return response - - -def _complete_gradient_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key: Final = ctx.api_key - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_base = litellm.api_base or api_base - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="gradient_ai", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) - - -def _complete_gdc(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.gdc_key or get_secret_str("GDC_API_KEY") or litellm.api_key - api_base = api_base or litellm.gdc_api_base or get_secret_str("GDC_API_BASE") or litellm.api_base - - return base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=gdc_transformation, - ) - - -def _complete_bytez(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.bytez_key or get_secret_str("BYTEZ_API_KEY") or litellm.api_key - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=bytez_transformation, - ) - - return response - - -def _complete_lemonade(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=lemonade_transformation, - ) - - return response - - -def _complete_ovhcloud(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers: Final = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - api_key = api_key or litellm.ovhcloud_key or get_secret_str("OVHCLOUD_API_KEY") or litellm.api_key - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("OVHCLOUD_API_BASE") - or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" - ) - - response: Final = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=ovhcloud_transformation, - ) - - return response - - -def _custom_api_first_output(resp: httpx.Response | None) -> str: - return resp.json()["data"][0]["output"][0] - - -def _complete_custom(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - api_base: Final = ctx.api_base - headers: Final = ctx.headers - kwargs: Final = ctx.kwargs - max_tokens: Final = ctx.max_tokens - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - temperature: Final = ctx.temperature - top_p: Final = ctx.top_p - - url: Final = litellm.api_base or api_base or "" - if url is None or url == "": - raise ValueError("api_base not set. Set api_base or litellm.api_base for custom endpoints") - - """ - assume input to custom LLM api bases follow this format: - resp = litellm.module_level_client.post( - api_base, - json={ - 'model': 'meta-llama/Llama-2-13b-hf', # model name - 'params': { - 'prompt': ["The capital of France is P"], - 'max_tokens': 32, - 'temperature': 0.7, - 'top_p': 1.0, - 'top_k': 40, - } - } - ) - - """ - prompt: Final = " ".join([message["content"] for message in messages]) - resp: Final = litellm.module_level_client.post( - url, - headers=headers, - json={ - "model": model, - "params": { - "prompt": [prompt], - "max_tokens": max_tokens, - "temperature": temperature, - "top_p": top_p, - "top_k": kwargs.get("top_k"), - }, - **kwargs.get("extra_body", {}), - }, - ) - """ - assume all responses from custom api_bases of this format: - { - 'data': [ - { - 'prompt': 'The capital of France is P', - 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], - 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], - 'message': 'ok' - } - ] - } - """ - string_response: Final = _custom_api_first_output(resp) - ## RESPONSE OBJECT - model_response.choices[0].message.content = string_response - model_response.created = int(time.time()) - model_response.model = model - return model_response - - -def _complete_custom_providers( - ctx: CompletionDispatchContext, -) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base: Final = ctx.api_base - api_key: Final = ctx.api_key - client: Final = ctx.client - custom_llm_provider: Final = ctx.custom_llm_provider - custom_prompt_dict: Final = ctx.custom_prompt_dict - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logger_fn: Final = ctx.logger_fn - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] - - if custom_handler is None: - raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) - - ## ROUTE LLM CALL ## - handler_fn: Final = custom_chat_llm_router(async_fn=acompletion, stream=stream, custom_llm=custom_handler) - - headers = headers or litellm.headers or {} - - ## CALL FUNCTION - response: Final = handler_fn( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - ) - if stream is True: - return CustomStreamWrapper( - completion_stream=response, - model=model, - custom_llm_provider=custom_llm_provider, - logging_obj=logging, - ) - - return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract - - -def _complete_langgraph(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - from litellm.llms.langgraph.chat.transformation import LangGraphConfig - - ( - api_base, - api_key, - ) = LangGraphConfig().get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) - - headers = headers or litellm.headers - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - -def _complete_microsoft_365_copilot(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - from litellm.llms.microsoft_365_copilot.chat.handler import acompletion, completion - from litellm.llms.microsoft_365_copilot.common_utils import as_secret_fields - - messages: Final[Sequence[AllMessageValues]] = cast( # cast-ok: dispatch messages are prevalidated - Sequence[AllMessageValues], - ctx.messages, - ) - optional_params_value: Final[object] = cast( # cast-ok: dispatch params are validated below - object, - ctx.optional_params, - ) - optional_params: Final = TypeAdapter(Mapping[str, object]).validate_python(optional_params_value) - kwargs_value: Final[object] = cast( # cast-ok: dispatch kwargs are validated below - object, - ctx.kwargs, - ) - kwargs: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs_value) - secret_fields: Final = as_secret_fields(kwargs.get("secret_fields")) - client: Final = _dispatch_client_http(ctx) - if ctx.acompletion: - async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None - return acompletion( - model=ctx.model, - messages=messages, - api_key=ctx.api_key, - litellm_params=ctx.litellm_params, - optional_params=optional_params, - secret_fields=secret_fields, - stream=ctx.stream is True, - timeout=ctx.timeout, - logging_obj=ctx.logging, - client=async_client, - shared_session=ctx.shared_session, - ) - sync_client: Final = client if isinstance(client, HTTPHandler) else None - return completion( - model=ctx.model, - messages=messages, - api_key=ctx.api_key, - litellm_params=ctx.litellm_params, - optional_params=optional_params, - secret_fields=secret_fields, - stream=ctx.stream is True, - timeout=ctx.timeout, - logging_obj=ctx.logging, - client=sync_client, - ) - - -def _complete_langflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - acompletion: Final = ctx.acompletion - api_base = ctx.api_base - api_key = ctx.api_key - client: Final = _dispatch_client_http(ctx) - custom_llm_provider: Final = ctx.custom_llm_provider - headers = ctx.headers - litellm_params: Final = ctx.litellm_params - logging: Final = ctx.logging - messages: Final = ctx.messages - model: Final = ctx.model - model_response: Final = ctx.model_response - optional_params: Final = ctx.optional_params - shared_session: Final = ctx.shared_session - stream: Final = ctx.stream - timeout: Final = ctx.timeout - - from litellm.llms.langflow.chat.transformation import LangFlowConfig - - ( - api_base, - api_key, - ) = LangFlowConfig().get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) - - headers = headers or litellm.headers - - return base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - - -def _complete_databricks_agent(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: - from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig - - api_base, api_key = DatabricksAgentConfig.resolve_api_base_and_key( - api_base=ctx.api_base or litellm.api_base, - api_key=ctx.api_key or litellm.api_key, - ) - return base_llm_http_handler.completion( - model=ctx.model, - stream=ctx.stream, - messages=ctx.messages, - acompletion=ctx.acompletion, - api_base=api_base, - model_response=ctx.model_response, - optional_params=ctx.optional_params, - litellm_params={**ctx.litellm_params, "api_base": api_base}, - shared_session=ctx.shared_session, - custom_llm_provider=ctx.custom_llm_provider, - timeout=ctx.timeout, - headers=ctx.headers or litellm.headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=ctx.logging, - client=_dispatch_client_http(ctx), - ) +_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str]) @tracer.wrap() @@ -5498,7 +869,7 @@ def completion( top_p: float | None = None, n: int | None = None, stream: bool | None = None, - stream_options: dict | None = None, + stream_options: dict[str, object] | None = None, stop=None, max_completion_tokens: int | None = None, max_tokens: int | None = None, @@ -5507,12 +878,12 @@ def completion( audio: ChatCompletionAudioParam | None = None, presence_penalty: float | None = None, frequency_penalty: float | None = None, - logit_bias: dict | None = None, + logit_bias: dict[str, float] | None = None, user: str | None = None, # openai v1.0+ new params reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None, verbosity: Literal["low", "medium", "high"] | None = None, - response_format: dict | type[BaseModel] | None = None, + response_format: dict[str, object] | type[BaseModel] | None = None, seed: int | None = None, tools: list | None = None, tool_choice: str | dict | None = None, @@ -5521,15 +892,15 @@ def completion( parallel_tool_calls: bool | None = None, web_search_options: OpenAIWebSearchOptions | None = None, include_server_side_tool_invocations: bool | None = None, - deployment_id=None, - extra_headers: dict | None = None, + deployment_id: str | None = None, + extra_headers: dict[str, str] | None = None, safety_identifier: str | None = None, service_tier: str | None = None, store: bool | None = None, prompt_cache_key: str | None = None, prompt_cache_retention: str | None = None, # soon to be deprecated params by OpenAI - functions: list | None = None, + functions: list[Mapping[str, object]] | None = None, function_call: str | None = None, # set api_base, api_version, api_key base_url: str | None = None, @@ -5604,14 +975,9 @@ def completion( ######### unpacking kwargs ##################### args: Final = _locals_snapshot(locals()) - # Set by the responses->completion fallback so completion() does not bridge - # back to the Responses API: that round-trip mutually recurses forever for a - # model whose model_cost mode is "responses" but whose provider has no - # Responses API config (get_provider_responses_api_config -> None). - skip_responses_api_bridge: Final = kwargs.pop("_skip_responses_api_bridge", False) - control_options: Final = _resolve_control_options(kwargs, model) skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) + skip_responses_api_bridge: Final = kwargs.pop("_skip_responses_api_bridge", False) is True if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp @@ -5662,82 +1028,12 @@ def completion( enable_json_schema_validation=enable_json_schema_validation, **kwargs, ) - api_base = kwargs.get("api_base", None) - mock_response: Final[MOCK_RESPONSE_TYPE | None] = kwargs.get("mock_response", None) - mock_tool_calls: Final = kwargs.get("mock_tool_calls", None) - mock_timeout: Final = cast(bool | None, kwargs.get("mock_timeout", None)) - force_timeout: Final = kwargs.get("force_timeout", 600) ## deprecated - logger_fn: Final = kwargs.get("logger_fn", None) - verbose: Final = kwargs.get("verbose", False) - custom_llm_provider = kwargs.get("custom_llm_provider", None) + custom_llm_provider: Final = kwargs.get("custom_llm_provider", None) litellm_logging_obj: Final = kwargs.get("litellm_logging_obj", None) - id: Final = kwargs.get("id", None) - metadata: Final = kwargs.get("metadata", None) - model_info: Final = kwargs.get("model_info", None) - proxy_server_request: Final = kwargs.get("proxy_server_request", None) fallbacks = kwargs.get("fallbacks", None) - provider_specific_header: Final = cast( - ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, - kwargs.get("provider_specific_header", None), - ) - headers = kwargs.get("headers", None) or extra_headers - - ensure_alternating_roles: Final = cast( # cast-ok: read from untyped kwargs - bool | None, kwargs.get("ensure_alternating_roles", None) - ) - user_continue_message: Final = cast( # cast-ok: read from untyped kwargs - ChatCompletionUserMessage | None, kwargs.get("user_continue_message", None) - ) - assistant_continue_message: Final = cast( # cast-ok: read from untyped kwargs - ChatCompletionAssistantMessage | None, kwargs.get("assistant_continue_message", None) - ) - headers = {} if headers is None else dict(headers) - if extra_headers is not None: - headers.update(extra_headers) - # Inject proxy auth headers if configured - if litellm.proxy_auth is not None: - try: - proxy_headers: Final = _proxy_auth_headers(litellm.proxy_auth) - headers.update(proxy_headers) - except Exception as e: - verbose_logger.warning("Failed to get proxy auth headers: %s", e) - num_retries: Final = kwargs.get( - "num_retries", None - ) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor. - max_retries = kwargs.get("max_retries", None) - cooldown_time: Final = kwargs.get("cooldown_time", None) - context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", None) - organization: Final = kwargs.get("organization", None) - ### VERIFY SSL ### - ssl_verify: Final = kwargs.get("ssl_verify", None) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) - ### CUSTOM PROMPT TEMPLATE ### - initial_prompt_value: Final = kwargs.get("initial_prompt_value", None) - roles: Final = kwargs.get("roles", None) - final_prompt_value: Final = kwargs.get("final_prompt_value", None) - bos_token: Final = kwargs.get("bos_token", None) - eos_token: Final = kwargs.get("eos_token", None) - preset_cache_key: Final = kwargs.get("preset_cache_key", None) - hf_model_name: Final = kwargs.get("hf_model_name", None) - supports_system_message: Final = kwargs.get("supports_system_message", None) - base_model: Final = kwargs.get("base_model", None) or ( - model_info.get("base_model") if isinstance(model_info, dict) else None - ) - ### DISABLE FLAGS ### - disable_add_transform_inline_image_block: Final = kwargs.get("disable_add_transform_inline_image_block", None) - ### TEXT COMPLETION CALLS ### - text_completion: Final = kwargs.get("text_completion", False) - atext_completion: Final = kwargs.get("atext_completion", False) - ### ASYNC CALLS ### - acompletion: Final = kwargs.get("acompletion", False) - client: Final = kwargs.get("client", None) - ### Admin Controls ### - no_log: Final = kwargs.get("no-log", False) + ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None) + user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None) + assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None) ### PROMPT MANAGEMENT ### prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) prompt_variables: Final = cast(dict | None, kwargs.get("prompt_variables", None)) @@ -5756,7 +1052,6 @@ def completion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, ) - from litellm.types.llms.openai import AllMessageValues AnthropicCacheControlHook.maybe_seed_default_injection_points( non_default_params=non_default_params, @@ -5776,7 +1071,7 @@ def completion( ( model, messages, - optional_params, + _, ) = litellm_logging_obj.get_chat_completion_prompt( model=model, messages=messages, @@ -5797,14 +1092,6 @@ def completion( ) try: - if base_url is not None: - api_base = base_url - is_router_call: Final = any("model_group" in (kwargs.get(k) or ()) for k in ("metadata", "litellm_metadata")) - if is_router_call: - max_retries = 0 - elif num_retries is not None: - max_retries = num_retries - logging: Final[LiteLLMLoggingObj] = cast(LiteLLMLoggingObj, litellm_logging_obj) fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime @@ -5815,788 +1102,57 @@ def completion( return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type deployments=deployments, **args ) - if litellm.model_alias_map and model in litellm.model_alias_map: - model = litellm.model_alias_map[ - model - ] # update the model to the actual value if an alias has been passed in - model_response: Final = ModelResponse() - setattr(model_response, "usage", litellm.Usage()) - if ( - kwargs.get("azure", False) is True - ): # don't remove flag check, to remain backwards compatible for repos like Codium - custom_llm_provider = "azure" - if deployment_id is not None: # azure llms - model = deployment_id - custom_llm_provider = "azure" - _supplemental_provider_params: Final = { - **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, - "extra_headers": extra_headers, - } - requested_model: Final = model - model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + return execute_chat( model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, - litellm_params=( - GenericLiteLLMParams.model_validate(_supplemental_provider_params) - if _supplemental_provider_params - else None - ), - ) - _record_agentic_loop_params( - logging_obj=logging, - requested_model=requested_model, - custom_llm_provider=custom_llm_provider, - owns_agentic_loop=not skip_responses_api_bridge, - ) - - deployment_model_info: Final = str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ) - ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name - responses_api_model_info, model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - api_base=api_base, - deployment_model_info=deployment_model_info, - ) - routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict - drop_unsupported_reasoning: Final = ( - litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True - ) - reasoning_model_group: Final = model_group_from_metadata( - routing_kwargs.get("metadata"), routing_kwargs.get("litellm_metadata") - ) - requested_output_config: Final = as_str_mapping(non_default_params.get("output_config")) - chat_reasoning: Final = resolve_chat_reasoning( - reasoning_effort, - thinking, - requested_output_config, - model=model, - custom_llm_provider=custom_llm_provider, - deployment_model_info=routing_kwargs.get("model_info"), - model_group=reasoning_model_group, - drop_params=drop_unsupported_reasoning, - ) - resolved_reasoning_effort: Final = chat_reasoning.effort.value - resolved_output_config: Final = chat_reasoning.output_config - provider_params: Final = ( - non_default_params - if resolved_output_config is requested_output_config - else { - **{key: value for key, value in non_default_params.items() if key != "output_config"}, - **({"output_config": resolved_output_config} if resolved_output_config is not None else {}), - } - ) - - if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): - tools = _drop_anthropic_only_tool_keys(tools=tools) - - if provider_specific_header is not None: - headers.update( - ProviderSpecificHeaderUtils.get_provider_specific_headers( - provider_specific_header=provider_specific_header, - custom_llm_provider=custom_llm_provider, - ) - ) - - if model_response is not None and hasattr(model_response, HIDDEN_PARAMS_ATTR): - model_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) - ) - model_response_hidden_params["custom_llm_provider"] = custom_llm_provider - model_response_hidden_params["region_name"] = kwargs.get( - "aws_region_name", None - ) # support region-based pricing for bedrock - - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### - custom_prompt_dict = {} - if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: - custom_prompt_dict = {model: {}} - if initial_prompt_value: - custom_prompt_dict[model]["initial_prompt_value"] = initial_prompt_value - if roles: - custom_prompt_dict[model]["roles"] = roles - if final_prompt_value: - custom_prompt_dict[model]["final_prompt_value"] = final_prompt_value - if bos_token: - custom_prompt_dict[model]["bos_token"] = bos_token - if eos_token: - custom_prompt_dict[model]["eos_token"] = eos_token - - messages = update_messages_with_model_file_ids( messages=messages, - model_id=(kwargs.get("model_info") or {}).get("id", None), - model_file_id_mapping=cast( - dict[str, dict[str, str]], - kwargs.get("model_file_id_mapping") or {}, - ), - ) - - provider_config: BaseConfig | None = None - if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=LlmProviders(custom_llm_provider), - base_model=base_model, - ) - - if provider_config is not None: - translated_messages: Final = list( - provider_config.translate_developer_role_to_system_role( - messages=messages, - model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - model_info=deployment_model_info, - ) - ) - history_for_route: Final = _history_without_unanswered_tool_search_calls( - translated_messages, custom_llm_provider, provider_config - ) - messages = ( # rebind-ok: completion() rewrites messages step by step for the provider - translated_messages if history_for_route is None else list(history_for_route) - ) - - if ( - supports_system_message is not None - and isinstance(supports_system_message, bool) - and supports_system_message is False - ): - messages = map_system_message_pt(messages=messages) - - if dynamic_api_key is not None: - api_key = dynamic_api_key - # check if user passed in any of the OpenAI optional params - bridges_to_responses_api: Final = ( - responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge - ) - allowed_openai_params: Final[list[str] | None] = ( - [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] - if bridges_to_responses_api - or reasoning_effort_needs_allowing( - chat_reasoning.effort, - model=model, - custom_llm_provider=custom_llm_provider, - base_model=base_model if isinstance(base_model, str) else None, - ) - else kwargs.get("allowed_openai_params") - ) - optional_param_args: Final = { - "functions": functions, - "function_call": function_call, - "temperature": temperature, - "top_p": top_p, - "n": n, - "stream": stream, - "stream_options": stream_options, - "stop": stop, - "max_tokens": max_tokens, - "max_completion_tokens": max_completion_tokens, - "modalities": modalities, - "prediction": prediction, - "audio": audio, - "presence_penalty": presence_penalty, - "frequency_penalty": frequency_penalty, - "logit_bias": logit_bias, - "user": user, - # params to identify the model - "model": model, - "custom_llm_provider": custom_llm_provider, - "response_format": response_format, - "seed": seed, - "tools": tools, - "tool_choice": tool_choice, - "max_retries": max_retries, - "logprobs": logprobs, - "top_logprobs": top_logprobs, - "api_version": api_version, - "parallel_tool_calls": parallel_tool_calls, - "messages": messages, - "reasoning_effort": resolved_reasoning_effort, - "verbosity": verbosity, - "thinking": chat_reasoning.thinking, - "web_search_options": web_search_options, - "include_server_side_tool_invocations": ( - include_server_side_tool_invocations - if include_server_side_tool_invocations is not None - else kwargs.get("include_server_side_tool_invocations") - ), - "safety_identifier": safety_identifier, - "service_tier": service_tier, - "store": store, - "prompt_cache_key": prompt_cache_key, - "prompt_cache_retention": prompt_cache_retention, - "allowed_openai_params": allowed_openai_params, - "base_model": base_model, - } - provider_reasoning_effort: Final = reasoning_effort_with_summary_alias( - resolved_reasoning_effort, - provider_params, - accepts_summary=provider_config is not None - and provider_config.accepts_reasoning_summary_in_reasoning_effort, - ) - - mapping_args: Final = MappingProxyType({**optional_param_args, "reasoning_effort": provider_reasoning_effort}) - mapping_special_params: Final = MappingProxyType(dict(provider_params)) - - def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: - return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) - - validated_model_info: Final = ( - _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None - ) - deployment_base_model: Final = base_model if isinstance(base_model, str) else None - optional_params = optional_params_for_model_info(validated_model_info) - fakes_stream: Final = partial( - _completion_fakes_stream, - model, - custom_llm_provider, - provider_config, - base_model=deployment_base_model, - bridges_to_responses_api=bridges_to_responses_api is True, - ) - resolve_timeout: Final = partial( - CompletionTimeout.resolve, - timeout, - kwargs, - custom_llm_provider, - global_timeout=get_configured_request_timeout(), - supports_httpx_timeout=supports_httpx_timeout, - ) - streams: Final = stream is True - - def timeout_for_optional_params(attempt_params: Mapping[str, object]) -> float | httpx.Timeout: - if not streams: - return resolve_timeout(stream_kind=StreamKind.NOT_STREAMED) - return resolve_timeout(stream_kind=StreamKind.FAKED if fakes_stream(attempt_params) else StreamKind.LIVE) - - ### TIMEOUT LOGIC ### - timeout = timeout_for_optional_params(optional_params) - logging.record_applied_timeout(timeout) - processed_non_default_params: Final = pre_process_non_default_params( - model=model, - passed_params=optional_param_args, - special_params=provider_params, - custom_llm_provider=custom_llm_provider, - additional_drop_params=kwargs.get("additional_drop_params"), - remove_sensitive_keys=True, - add_provider_specific_params=True, - provider_config=provider_config, - ) - - if litellm.add_function_to_prompt and optional_params.get( - "functions_unsupported_model", None - ): # if user opts to add it to prompt, when API doesn't support function calling - functions_unsupported_model: Final[list[object]] = TypeAdapter(list[object]).validate_python( - optional_params.pop("functions_unsupported_model") - ) - messages = function_call_prompt( - messages=messages, - functions=functions_unsupported_model, - model=model, - llm_provider=custom_llm_provider, - drop_params=drop_unsupported_reasoning, - ) - - readable_messages: Final = _messages_readable_by_model( - messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List - model=model, - custom_llm_provider=custom_llm_provider, - model_info=validated_model_info, - base_model=deployment_base_model, - drop_params=drop_unsupported_reasoning, - ensure_alternating_roles=ensure_alternating_roles, - user_continue_message=user_continue_message, - assistant_continue_message=assistant_continue_message, - ) - messages = ( # rebind-ok: completion() rewrites messages step by step - messages if readable_messages is None else list(readable_messages) - ) - - # For logging - save the values of the litellm-specific params passed in - requested_litellm_params: Final = get_litellm_params( - acompletion=acompletion, + timeout=timeout, + temperature=temperature, + top_p=top_p, + n=n, + stream=stream, + stream_options=stream_options, + stop=stop, + max_completion_tokens=max_completion_tokens, + max_tokens=max_tokens, + modalities=modalities, + prediction=prediction, + audio=audio, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + logit_bias=logit_bias, + user=user, + reasoning_effort=reasoning_effort, + verbosity=verbosity, + response_format=response_format, + seed=seed, + tools=tools, + tool_choice=tool_choice, + logprobs=logprobs, + top_logprobs=top_logprobs, + parallel_tool_calls=parallel_tool_calls, + web_search_options=web_search_options, + include_server_side_tool_invocations=include_server_side_tool_invocations, + deployment_id=deployment_id, + extra_headers=extra_headers, + safety_identifier=safety_identifier, + service_tier=service_tier, + store=store, + prompt_cache_key=prompt_cache_key, + prompt_cache_retention=prompt_cache_retention, + functions=functions, + function_call=function_call, + base_url=base_url, + api_version=api_version, api_key=api_key, - force_timeout=force_timeout, - logger_fn=logger_fn, - verbose=verbose, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - litellm_call_id=kwargs.get("litellm_call_id", None), - model_alias_map=litellm.model_alias_map, - completion_call_id=id, - metadata=metadata, - model_info=model_info, - proxy_server_request=proxy_server_request, - preset_cache_key=preset_cache_key, - no_log=no_log, - cost_per_second=cost_per_second, - input_cost_per_second=input_cost_per_second, - input_cost_per_token=input_cost_per_token, - output_cost_per_second=output_cost_per_second, - output_cost_per_token=output_cost_per_token, - cooldown_time=cooldown_time, - text_completion=kwargs.get("text_completion"), - azure_ad_token_provider=kwargs.get("azure_ad_token_provider"), - user_continue_message=kwargs.get("user_continue_message"), - base_model=base_model, - litellm_trace_id=kwargs.get("litellm_trace_id"), - litellm_session_id=kwargs.get("litellm_session_id"), - hf_model_name=hf_model_name, - custom_prompt_dict=custom_prompt_dict, - litellm_metadata=kwargs.get("litellm_metadata"), - disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, - drop_params=kwargs.get("drop_params"), + thinking=thinking, + shared_session=shared_session, + kwargs=kwargs, + non_default_params=non_default_params, + control_options=control_options, + skip_responses_api_bridge=skip_responses_api_bridge, prompt_id=prompt_id, prompt_variables=prompt_variables, - ssl_verify=ssl_verify, - merge_reasoning_content_in_choices=kwargs.get("merge_reasoning_content_in_choices", None), - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - api_version=api_version, - azure_ad_token=kwargs.get("azure_ad_token"), - tenant_id=kwargs.get("tenant_id"), - client_id=kwargs.get("client_id"), - client_secret=kwargs.get("client_secret"), - azure_username=kwargs.get("azure_username"), - azure_password=kwargs.get("azure_password"), - azure_scope=kwargs.get("azure_scope"), - max_retries=max_retries, - timeout=timeout, - litellm_request_debug=kwargs.get("litellm_request_debug", False), - tpm=kwargs.get("tpm"), - rpm=kwargs.get("rpm"), - use_xai_oauth=kwargs.get("use_xai_oauth", False), - chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"), - gigachat_scope=kwargs.get("gigachat_scope"), - gigachat_auth_url=kwargs.get("gigachat_auth_url"), - gigachat_access_token=kwargs.get("gigachat_access_token"), - **{ - key: kwargs[key] - for key in ( - *AWS_CREDENTIAL_KWARGS_KEYS, - *ANTHROPIC_WIF_KWARGS_KEYS, - *OPENAI_WIF_KWARGS_KEYS, - *OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, - "github_copilot_auth_type", - "github_copilot_host", - "github_copilot_user_session", - PROVIDER_AFFINITY_HEADER_KWARG_KEY, - "fireworks_forward_user_id", - "responses_api_version", - ) - if key in kwargs - }, ) - litellm_params: Final = with_control_options(requested_litellm_params, control_options) - affinity_api_base: Final = ( - str(client.base_url) - if isinstance(client, openai.OpenAI | openai.AsyncOpenAI) - else get_openai_credentials(api_base=api_base).api_base - if custom_llm_provider == "openai" - else api_base - ) - default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) - if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None: - try: - headers = add_provider_affinity_header( - headers=headers or litellm.headers or MappingProxyType({}), - litellm_params=MappingProxyType( - { - "provider_affinity_header": litellm_params.get("provider_affinity_header"), - "litellm_session_id": kwargs.get("litellm_session_id"), - "session_id": kwargs.get("session_id"), - "metadata": metadata, - "litellm_metadata": kwargs.get("litellm_metadata"), - } - ), - default_header=default_affinity_header, - ) - except ValueError as affinity_error: - raise litellm.BadRequestError( - message=str(affinity_error), model=model, llm_provider=custom_llm_provider - ) from affinity_error - logging.update_environment_variables( - model=model, - user=user, - optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. - litellm_params=litellm_params, - custom_llm_provider=custom_llm_provider, - model_id=optional_params.get("model_id"), - ) - if mock_response is not None or mock_tool_calls or mock_timeout: - kwargs.pop("mock_timeout", None) # remove for any fallbacks triggered - return mock_completion( - model, - messages, - stream=stream, - n=n, - mock_response=mock_response, - mock_tool_calls=mock_tool_calls, - logging=logging, - acompletion=acompletion, - mock_delay=kwargs.get("mock_delay", None), - custom_llm_provider=custom_llm_provider, - mock_timeout=mock_timeout, - timeout=timeout, - prompt_tokens=admission_input_tokens( - cast(Mapping[str, object], kwargs) # cast-ok: completion's **kwargs is untyped - ), - ) - - ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map - # Only run the second bridge check if the first one didn't already - # detect responses mode (e.g. via the "responses/" prefix). The second - # check handles cases like gpt-5.4+ with tools+reasoning_effort or - # reasoningSummary/reasoning_summary without tools (AI SDK) that the first - # (early) check doesn't cover. - _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params) - if responses_api_model_info.get("mode") != "responses": - responses_api_model_info, model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - tools=tools, - reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")), - reasoning_summary=_reasoning_summary_for_bridge, - api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), - ) - - # Use base_model (the true underlying model) for Azure model-type - # detection when the deployment name differs from the model name. - _azure_detection_model: Final = base_model or model - - if responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge: - from litellm.completion_extras import responses_api_bridge - - optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) - - if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict): - from litellm.responses.utils import openai_bridge_reasoning - - optional_params["reasoning_effort"] = openai_bridge_reasoning( - requested=resolved_reasoning_effort, - effort_kept="reasoning_effort" in optional_params, - summary_alias=rs_val, - model=model, - drop_params=drop_unsupported_reasoning, - additional_drop_params=routing_kwargs.get("additional_drop_params"), - ) - elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: - optional_params["reasoning_effort"] = resolved_reasoning_effort - elif rs_val is not None: - eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) - if isinstance(eff, dict): - optional_params["reasoning_effort"] = {**eff, "summary": rs_val} - elif eff is not None: - optional_params["reasoning_effort"] = { - "effort": eff, - "summary": rs_val, - } - else: - optional_params["reasoning_effort"] = {"summary": rs_val} - - return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, # pass AsyncOpenAI, OpenAI client - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - cache=kwargs.get("cache"), - ) - elif (custom_llm_provider == "openai" and OpenAIGPT5Config.is_model_gpt_5_model(model)) or ( - custom_llm_provider == "azure" - and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(_azure_detection_model) - ): - optional_params, _ = strip_reasoning_summary_aliases_from_optional_params(optional_params) - - _dispatch_ctx: Final = CompletionDispatchContext( - _azure_detection_model=_azure_detection_model, - acompletion=acompletion, - api_base=api_base, - api_key=api_key, - api_version=api_version, - client=client, - custom_llm_provider=custom_llm_provider, - custom_prompt_dict=custom_prompt_dict, - extra_headers=extra_headers, - headers=headers, - hf_model_name=hf_model_name, - kwargs=kwargs, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging=logging, - max_retries=max_retries, - max_tokens=max_tokens, - messages=messages, - metadata=metadata, - model=model, - model_response=model_response, - optional_params=optional_params, - optional_params_for_model_info=optional_params_for_model_info, - organization=organization, - provider_config=provider_config, - request_params=MappingProxyType( - _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **provider_params}) - ), - shared_session=shared_session, - stream=stream, - temperature=temperature, - text_completion=text_completion, - timeout=timeout, - timeout_for_optional_params=timeout_for_optional_params, - top_p=top_p, - ) - if custom_llm_provider == "azure": - # azure configs - ## check dynamic params ## - response = _complete_azure(_dispatch_ctx) - elif custom_llm_provider == "azure_text": - # azure configs - response = _complete_azure_text(_dispatch_ctx) - elif custom_llm_provider == "deepseek": - ## COMPLETION CALL - - response = _complete_deepseek(_dispatch_ctx) - - elif custom_llm_provider == "azure_ai": - response = _complete_azure_ai(_dispatch_ctx) - elif ( - custom_llm_provider == "text-completion-openai" - or "ft:babbage-002" in model - or "ft:davinci-002" in model # support for finetuned completion models - or custom_llm_provider in litellm.openai_text_completion_compatible_providers - and kwargs.get("text_completion") is True - ): - response = _complete_text_completion_openai(_dispatch_ctx) - elif custom_llm_provider == "fireworks_ai": - ## COMPLETION CALL - response = _complete_fireworks_ai(_dispatch_ctx) - elif custom_llm_provider == "together_ai": - response = _complete_together_ai(_dispatch_ctx) - elif custom_llm_provider == "heroku": - response = _complete_heroku(_dispatch_ctx) - - elif custom_llm_provider == "ragflow": - ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths - response = _complete_ragflow(_dispatch_ctx) - elif custom_llm_provider == "scaledown": - response = _complete_scaledown(_dispatch_ctx) # rebind-ok: shared provider return - elif custom_llm_provider == "xai": - ## COMPLETION CALL - response = _complete_xai(_dispatch_ctx) - elif custom_llm_provider == "groq": - response = _complete_groq(_dispatch_ctx) - elif custom_llm_provider == "opencode_go": - response = _complete_opencode_go( # rebind-ok: common dispatcher finalizes the selected provider result - _dispatch_ctx - ) - elif custom_llm_provider == "bedrock_mantle": - response = _complete_bedrock_mantle(_dispatch_ctx) - elif custom_llm_provider == "a2a": - # A2A (Agent-to-Agent) Protocol - # Resolve agent configuration from registry if model format is "a2a/" - response = _complete_a2a(_dispatch_ctx) - elif custom_llm_provider == "gigachat": - # GigaChat - Sber AI's LLM (Russia) - response = _complete_gigachat(_dispatch_ctx) - - elif custom_llm_provider == "sap": - response = _complete_sap(_dispatch_ctx) - elif custom_llm_provider == "aiohttp_openai": - # NEW aiohttp provider for 10-100x higher RPS - response = _complete_aiohttp_openai(_dispatch_ctx) - elif custom_llm_provider == "cometapi": - response = _complete_cometapi(_dispatch_ctx) - elif custom_llm_provider == "minimax": - response = _complete_minimax(_dispatch_ctx) - elif custom_llm_provider == "hosted_vllm": - response = _complete_hosted_vllm(_dispatch_ctx) - elif custom_llm_provider == "edenai": - response = _complete_edenai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch - elif custom_llm_provider == "fal_ai": - response = _complete_fal_ai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch - elif ( - # A known OpenAI model name only decides the route when nothing else - # resolved a provider. get_llm_provider() already maps these names to - # "openai", so a different value here was asked for explicitly (or came - # from a register_model entry), and the provider config built for it - # would be handed to the OpenAI handler. - (model in litellm.open_ai_chat_completion_models and custom_llm_provider in (None, "openai")) - or custom_llm_provider == "custom_openai" - or custom_llm_provider == "deepinfra" - or custom_llm_provider == "perplexity" - or custom_llm_provider == "nvidia_nim" - or custom_llm_provider == "cerebras" - or custom_llm_provider == "baseten" - or custom_llm_provider == "sambanova" - or custom_llm_provider == "volcengine" - or custom_llm_provider == "anyscale" - or custom_llm_provider == "openai" - or custom_llm_provider == "nebius" - or custom_llm_provider == "wandb" - or custom_llm_provider == "clarifai" - or custom_llm_provider == "bigmodel" - or custom_llm_provider in litellm.openai_compatible_providers - or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers - or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo - ): # allow user to make an openai call with a custom base - # note: if a user sets a custom base - we should ensure this works - # allow for the setting of dynamic and stateful api-bases - response = _complete_custom_openai(_dispatch_ctx) - - elif custom_llm_provider == "mistral": - response = _complete_mistral(_dispatch_ctx) - elif "replicate" in model or custom_llm_provider == "replicate" or model in litellm.replicate_models: - # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") - response = _complete_replicate(_dispatch_ctx) - elif "clarifai" in model or custom_llm_provider == "clarifai" or model in litellm.clarifai_models: - pass # Deprecated - handled in the openai compatible provider section above - elif custom_llm_provider == "anthropic_text": - response = _complete_anthropic_text(_dispatch_ctx) - elif custom_llm_provider == "anthropic": - response = _complete_anthropic(_dispatch_ctx) - elif custom_llm_provider == "nlp_cloud": - response = _complete_nlp_cloud(_dispatch_ctx) - elif custom_llm_provider == "aleph_alpha": - response = _complete_aleph_alpha(_dispatch_ctx) - elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": - response = _complete_cohere_chat(_dispatch_ctx) - elif custom_llm_provider == "maritalk": - response = _complete_maritalk(_dispatch_ctx) - elif custom_llm_provider == "amazon_nova": - response = _complete_amazon_nova(_dispatch_ctx) - elif custom_llm_provider == "huggingface": - response = _complete_huggingface(_dispatch_ctx) - elif custom_llm_provider == "oci": - response = _complete_oci(_dispatch_ctx) - elif custom_llm_provider == "compactifai": - response = _complete_compactifai(_dispatch_ctx) - elif custom_llm_provider == "oobabooga": - response = _complete_oobabooga(_dispatch_ctx) - elif custom_llm_provider == "databricks": - response = _complete_databricks(_dispatch_ctx) - - elif custom_llm_provider == "datarobot": - response = _complete_datarobot(_dispatch_ctx) - elif custom_llm_provider == "openrouter": - response = _complete_openrouter(_dispatch_ctx) - elif custom_llm_provider == "nadir": - response = _complete_nadir(_dispatch_ctx) # rebind-ok: mirrors sibling provider branches - elif custom_llm_provider == "vercel_ai_gateway": - response = _complete_vercel_ai_gateway(_dispatch_ctx) - elif custom_llm_provider == "palm": - raise ValueError( - "Palm was decommissioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" - ) - elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini": - response = _complete_vertex_ai_beta(_dispatch_ctx) - - elif custom_llm_provider == "vertex_ai": - response = _complete_vertex_ai(_dispatch_ctx) - elif custom_llm_provider == "predibase": - response = _complete_predibase(_dispatch_ctx) - elif custom_llm_provider == "text-completion-codestral": - response = _complete_text_completion_codestral(_dispatch_ctx) - elif custom_llm_provider == "text-completion-inception": - response = _complete_text_completion_inception(_dispatch_ctx) - elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): - # boto3 reads keys from .env - # sagemaker_chat: HF Messages API endpoints - # sagemaker_nova: Nova models on SageMaker (OpenAI-compatible) - response = _complete_sagemaker_chat(_dispatch_ctx) - elif custom_llm_provider == "sagemaker": - # boto3 reads keys from .env - response = _complete_sagemaker(_dispatch_ctx) - elif custom_llm_provider == "bedrock": - # boto3 reads keys from .env - response = _complete_bedrock(_dispatch_ctx) - elif custom_llm_provider == "watsonx": - response = _complete_watsonx(_dispatch_ctx) - elif custom_llm_provider == "watsonx_text": - response = _complete_watsonx_text(_dispatch_ctx) - elif custom_llm_provider == "vllm": - response = _complete_vllm(_dispatch_ctx) - elif custom_llm_provider == "ollama": - response = _complete_ollama(_dispatch_ctx) - - elif custom_llm_provider == "ollama_chat": - response = _complete_ollama_chat(_dispatch_ctx) - - elif custom_llm_provider == "triton": - response = _complete_triton(_dispatch_ctx) - elif custom_llm_provider == "cloudflare": - response = _complete_cloudflare(_dispatch_ctx) - - elif custom_llm_provider == "petals" or model in litellm.petals_models: - response = _complete_petals(_dispatch_ctx) - elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: - response = _complete_snowflake(_dispatch_ctx) - elif custom_llm_provider == "gradient_ai": - response = _complete_gradient_ai(_dispatch_ctx) - - elif custom_llm_provider == "gdc": - response = _complete_gdc(_dispatch_ctx) - elif custom_llm_provider == "bytez": - response = _complete_bytez(_dispatch_ctx) - elif custom_llm_provider == "lemonade": - response = _complete_lemonade(_dispatch_ctx) - - elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models: - response = _complete_ovhcloud(_dispatch_ctx) - - elif custom_llm_provider == "custom": - response = _complete_custom(_dispatch_ctx) - - elif custom_llm_provider in litellm._custom_providers: # Assume custom LLM provider - # Get the Custom Handler - response = _complete_custom_providers(_dispatch_ctx) - - elif custom_llm_provider == "langgraph": - # LangGraph - Agent Runtime Provider - response = _complete_langgraph(_dispatch_ctx) - - elif custom_llm_provider == "microsoft_365_copilot": - response = _complete_microsoft_365_copilot(_dispatch_ctx) # rebind-ok: shared provider return - - elif custom_llm_provider == "langflow": - # LangFlow - Visual AI Agent Platform - response = _complete_langflow(_dispatch_ctx) - - elif custom_llm_provider == "databricks_agent": - response = _complete_databricks_agent(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch - - else: - raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) - return response except Exception as e: ## Map to OpenAI Exception raise exception_type( @@ -9399,13 +3955,6 @@ async def ahealth_check( ####### HELPER FUNCTIONS ################ ## Set verbose to true -> ```litellm.set_verbose = True``` -def print_verbose(print_statement: object): - try: - verbose_logger.debug(print_statement) - if litellm.set_verbose: - print(print_statement) # noqa: T201 - except Exception: - pass def config_completion(**kwargs): diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 35136c41dc2..bc001b8d336 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -3,9 +3,11 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Itera from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable +import litellm from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.messages.native import handler as main +from litellm.native.route_plan import NativeMessages, NativeResponses, plan_messages_route from litellm.rust_bridge import detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target from litellm.rust_bridge.dispatch import PublicDispatch @@ -86,16 +88,20 @@ def _resolved(request: NativeCall) -> tuple[str, str | None]: def _target(request: NativeCall, model: str, provider: str | None) -> Target: - requested_model: Final = str(request.resolved["model"]) - model_info: Final = request.resolved.get("model_info") api_base: Final = request.resolved.get("api_base") - native_config: Final = main._native_messages_config( # pyright: ignore[reportPrivateUsage] # the handler's own passthrough lookup - model, provider, model_info, api_base if isinstance(api_base, str) else None + route: Final = plan_messages_route( + requested_model=str(request.resolved["model"]), + resolved_model=model, + custom_llm_provider=provider, + model_info=request.resolved.get("model_info"), + api_base=api_base if isinstance(api_base, str) else None, + compaction_requested=request.resolved.get("compaction") is not None, + force_chat_completions=bool(litellm.use_chat_completions_url_for_anthropic_messages), + stop_requested=bool(request.resolved.get("stop_sequences")), ) - if native_config is not None: + if isinstance(route, NativeMessages): return Target.MESSAGES - compacting: Final = request.resolved.get("compaction") is not None - if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice + if isinstance(route, NativeResponses): return Target.RESPONSES return Target.CHAT_COMPLETIONS diff --git a/litellm/native/chat.py b/litellm/native/chat.py new file mode 100644 index 00000000000..5335fc7af21 --- /dev/null +++ b/litellm/native/chat.py @@ -0,0 +1,1342 @@ +from collections.abc import Coroutine, Mapping, Sequence +from functools import partial +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Optional, + cast, # noqa: TID251 # casts moved verbatim from litellm/main.py +) + +import httpx +import openai +from pydantic import BaseModel, TypeAdapter + +import litellm +from litellm._logging import verbose_logger +from litellm.exceptions import LiteLLMUnknownProvider +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind +from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params +from litellm.litellm_core_utils.get_litellm_params import ( + AWS_CREDENTIAL_KWARGS_KEYS, + OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, + OPTIONAL_KWARGS_KEYS, + PROVIDER_AFFINITY_HEADER_KWARG_KEY, + with_control_options, +) +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_completion_messages, + messages_have_pdf_file_part, + messages_without_pdf_file_parts, + update_messages_with_model_file_ids, +) +from litellm.litellm_core_utils.prompt_templates.factory import function_call_prompt, map_system_message_pt +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header +from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +from litellm.llms.anthropic.common_utils import is_claude_tool_target +from litellm.llms.base_llm import BaseConfig +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config +from litellm.llms.openai.common_utils import get_openai_credentials +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route +from litellm.native.chat_mock import MOCK_RESPONSE_TYPE, mock_completion +from litellm.native.chat_providers.agents import ( + complete_a2a, + complete_langflow, + complete_langgraph, + complete_microsoft_365_copilot, + complete_ragflow, +) +from litellm.native.chat_providers.anthropic import complete_anthropic, complete_anthropic_text +from litellm.native.chat_providers.aws import ( + complete_amazon_nova, + complete_bedrock, + complete_bedrock_mantle, + complete_sagemaker, + complete_sagemaker_chat, +) +from litellm.native.chat_providers.azure import complete_azure, complete_azure_ai, complete_azure_text +from litellm.native.chat_providers.custom import complete_custom, complete_custom_providers +from litellm.native.chat_providers.gateways import ( + complete_cometapi, + complete_edenai, + complete_fal_ai, + complete_nadir, + complete_openrouter, + complete_vercel_ai_gateway, +) +from litellm.native.chat_providers.google import complete_gdc, complete_vertex_ai, complete_vertex_ai_beta +from litellm.native.chat_providers.model_labs import ( + complete_aleph_alpha, + complete_bytez, + complete_cohere_chat, + complete_compactifai, + complete_deepseek, + complete_fireworks_ai, + complete_groq, + complete_huggingface, + complete_maritalk, + complete_minimax, + complete_mistral, + complete_nlp_cloud, + complete_opencode_go, + complete_predibase, + complete_replicate, + complete_scaledown, + complete_text_completion_codestral, + complete_together_ai, + complete_xai, +) +from litellm.native.chat_providers.openai import ( + complete_aiohttp_openai, + complete_custom_openai, + complete_text_completion_inception, + complete_text_completion_openai, +) +from litellm.native.chat_providers.platforms import ( + complete_cloudflare, + complete_databricks, + complete_databricks_agent, + complete_datarobot, + complete_gigachat, + complete_gradient_ai, + complete_heroku, + complete_oci, + complete_ovhcloud, + complete_sap, + complete_snowflake, + complete_watsonx, + complete_watsonx_text, +) +from litellm.native.chat_providers.self_hosted import ( + complete_hosted_vllm, + complete_lemonade, + complete_ollama, + complete_ollama_chat, + complete_oobabooga, + complete_petals, + complete_triton, + complete_vllm, +) +from litellm.native.chat_providers.support import get_encoding +from litellm.native.request_support import ( + admission_input_tokens, + proxy_auth_headers, + register_custom_pricing_for_request, +) +from litellm.native.route_plan import NativeResponses, plan_chat_route +from litellm.responses.bridge_check import bridge_reasoning_effort, responses_api_bridge_check, str_keyed_mapping +from litellm.router_utils.reasoning_effort_capability import ( + model_group_from_metadata, + reasoning_effort_needs_allowing, + resolve_chat_reasoning, +) +from litellm.types.completion import CompletionDispatchContext +from litellm.types.litellm_params import ControlOptions +from litellm.types.llms.anthropic import AnthropicThinkingParam +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionAudioParam, + ChatCompletionModality, + ChatCompletionPredictionContentParam, + ChatCompletionUserMessage, + OpenAIWebSearchOptions, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import AgenticLoopParams, LlmProviders, ProviderSpecificHeader +from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS +from litellm.utils import ( + CustomStreamWrapper, + ModelResponse, + ProviderConfigManager, + exception_type, + get_litellm_params, + get_llm_provider, + get_optional_params, + is_explicitly_disabled_factory, + peek_reasoning_summary_aliases, + pre_process_non_default_params, + reasoning_effort_with_summary_alias, + strip_reasoning_summary_aliases_from_optional_params, + supports_httpx_timeout, + supports_pdf_input, +) + +if TYPE_CHECKING: + from aiohttp import ClientSession + + +async def resolve_dispatched_chat_response( + pending: Coroutine[object, object, ModelResponse | CustomStreamWrapper], +) -> ModelResponse | CustomStreamWrapper: + return await pending + + +_ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"}) + + +def _history_without_unanswered_tool_search_calls( + messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig +) -> tuple[object, ...] | None: + from litellm.responses.litellm_completion_transformation.tool_search import ( + chat_messages_without_tool_search_calls, + chat_route_rejects_unanswered_tool_calls, + unanswered_tool_search_call_ids, + ) + + if not chat_route_rejects_unanswered_tool_calls(custom_llm_provider, provider_config): + return None + unanswered_search_call_ids: Final = unanswered_tool_search_call_ids(messages) + if not unanswered_search_call_ids: + return None + return chat_messages_without_tool_search_calls(messages, unanswered_search_call_ids) + + +def _without_anthropic_only_tool_keys(tool: dict[str, object]) -> dict[str, object]: + kept: Final = {key: value for key, value in tool.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS} + function: Final = tool.get("function") + if not isinstance(function, dict): + return kept + return { + **kept, + "function": {key: value for key, value in function.items() if key not in _ANTHROPIC_ONLY_TOOL_KEYS}, + } + + +def _drop_anthropic_only_tool_keys(tools: list[dict[str, object]] | None) -> list[dict[str, object]] | None: + if tools is None: + return None + return [_without_anthropic_only_tool_keys(tool) if isinstance(tool, dict) else tool for tool in tools] + + +OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) + + +def _bridges_to_responses_api( + responses_api_model_info: Mapping[str, object], + model: str, + custom_llm_provider: str, + model_info: object, + api_base: str | None, +) -> bool: + if responses_api_model_info.get("mode") != "responses": + return False + from litellm.responses.main import resolve_responses_api_provider_config + + return isinstance( + plan_chat_route(resolve_responses_api_provider_config(model, custom_llm_provider, model_info, api_base)), + NativeResponses, + ) + + +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"}) +PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"}) + + +def _cost_map_pdf_input_support(model: str, custom_llm_provider: str) -> bool | None: + if custom_llm_provider in PROVIDERS_WITHOUT_DOCUMENT_INPUT or is_explicitly_disabled_factory( + model, custom_llm_provider, "supports_pdf_input" + ): + return False + if supports_pdf_input(model, custom_llm_provider): + return True + return None + + +def _deployment_pdf_input_support(model: str, base_model: str, custom_llm_provider: str) -> bool | None: + supports: Final = ( + _cost_map_pdf_input_support(model, custom_llm_provider), + _cost_map_pdf_input_support(base_model, custom_llm_provider), + ) + if False in supports: + return False + if True in supports: + return True + return None + + +def _declared_pdf_input_support( + model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None, base_model: str | None +) -> bool | None: + match model_info: + case {"supports_pdf_input": bool() as declared}: + return declared + case _ if base_model: + return _deployment_pdf_input_support(model, base_model, custom_llm_provider) + case _: + return _cost_map_pdf_input_support(model, custom_llm_provider) + + +def _pdf_input_unreadable( + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, +) -> bool: + match _declared_pdf_input_support(model, custom_llm_provider, model_info, base_model): + case False: + return True + case None: + return drop_params and custom_llm_provider in PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT + case True: + return False + + +def _messages_without_unreadable_pdfs( + messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, +) -> tuple[AllMessageValues, ...] | None: + if not messages_have_pdf_file_part(messages) or not _pdf_input_unreadable( + model, custom_llm_provider, model_info, base_model, drop_params + ): + return None + if not drop_params: + raise litellm.BadRequestError( + message=( + f"{custom_llm_provider}/{model} does not accept PDF input. " + "Remove the PDF, or set drop_params=True to send the request without it" + ), + model=model, + llm_provider=custom_llm_provider, + ) + readable_messages: Final = messages_without_pdf_file_parts(messages) + if not any(message["role"] == "user" for message in readable_messages): + raise litellm.BadRequestError( + message=( + f"{custom_llm_provider}/{model} does not accept PDF input, " + "and no user message is left once the PDF is dropped" + ), + model=model, + llm_provider=custom_llm_provider, + ) + return readable_messages + + +def _messages_readable_by_model( + messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object] | None, + base_model: str | None, + drop_params: bool, + ensure_alternating_roles: bool | None, + user_continue_message: ChatCompletionUserMessage | None, + assistant_continue_message: ChatCompletionAssistantMessage | None, +) -> tuple[AllMessageValues, ...] | None: + pdf_free_messages: Final = _messages_without_unreadable_pdfs( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + model_info=model_info, + base_model=base_model, + drop_params=drop_params, + ) + if pdf_free_messages is None: + return None + return tuple( + get_completion_messages( + messages=list(pdf_free_messages), + ensure_alternating_roles=ensure_alternating_roles or False, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + ) + + +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + +def _azure_completion_fakes_stream(model: str, base_model: str | None) -> bool: + if not litellm.AzureOpenAIO1Config().is_o_series_model(model=base_model or model): + return False + o_series_handler_config: Final = ProviderConfigManager.get_provider_chat_config( + model=model, provider=LlmProviders.AZURE + ) + return o_series_handler_config is not None and o_series_handler_config.should_fake_stream( + model=model, stream=True, custom_llm_provider=LlmProviders.AZURE.value + ) + + +def _completion_fakes_stream( + model: str, + custom_llm_provider: str | None, + provider_config: BaseConfig | None, + optional_params: Mapping[str, object], + *, + base_model: str | None, + bridges_to_responses_api: bool, +) -> bool: + if optional_params.get("fake_stream") is True: + return True + if custom_llm_provider is None: + return False + if bridges_to_responses_api: + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( + provider=custom_llm_provider, model=model + ) + return responses_config is not None and responses_config.should_fake_stream( + model=model, stream=True, custom_llm_provider=custom_llm_provider + ) + if ( + custom_llm_provider == LlmProviders.VERTEX_AI.value + and get_vertex_ai_model_route(model=model, litellm_params={"base_model": base_model}) + == VertexAIModelRoute.GEMMA + ): + return True + if isinstance(provider_config, litellm.AmazonAI21Config): + return True + if custom_llm_provider == LlmProviders.AZURE.value: + return _azure_completion_fakes_stream(model, base_model) + return provider_config is not None and provider_config.should_fake_stream( + model=model, stream=True, custom_llm_provider=custom_llm_provider + ) + + +def execute_chat( + *, + model: str, + messages: list, # mutable-ok: provider steps rebuild the message list, as in completion() + kwargs: dict, # mutable-ok: provider steps pop request-scoped keys, as in completion() + non_default_params: dict[str, object], # mutable-ok: seeded in place before the provider call + control_options: ControlOptions, + prompt_id: str | None, + prompt_variables: dict | None, # mutable-ok: forwarded to get_litellm_params as is + timeout: float | str | httpx.Timeout | openai.Timeout | None = None, + temperature: float | None = None, + top_p: float | None = None, + n: int | None = None, + stream: bool | None = None, + stream_options: Mapping[str, object] | None = None, + stop: str | Sequence[str] | None = None, + max_completion_tokens: int | None = None, + max_tokens: int | None = None, + modalities: Sequence[ChatCompletionModality] | None = None, + prediction: ChatCompletionPredictionContentParam | None = None, + audio: ChatCompletionAudioParam | None = None, + presence_penalty: float | None = None, + frequency_penalty: float | None = None, + logit_bias: Mapping[str, float] | None = None, + user: str | None = None, + reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None, + verbosity: Literal["low", "medium", "high"] | None = None, + response_format: Mapping[str, object] | type[BaseModel] | None = None, + seed: int | None = None, + tools: list[dict[str, object]] | None = None, # mutable-ok: provider steps rebuild tools, as in completion() + tool_choice: str | Mapping[str, object] | None = None, + logprobs: bool | None = None, + top_logprobs: int | None = None, + parallel_tool_calls: bool | None = None, + web_search_options: OpenAIWebSearchOptions | None = None, + include_server_side_tool_invocations: bool | None = None, + deployment_id: str | None = None, + extra_headers: dict[str, str] | None = None, # mutable-ok: forwarded to CompletionDispatchContext as is + safety_identifier: str | None = None, + service_tier: str | None = None, + store: bool | None = None, + prompt_cache_key: str | None = None, + prompt_cache_retention: str | None = None, + functions: Sequence[Mapping[str, object]] | None = None, + function_call: str | None = None, + base_url: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + thinking: AnthropicThinkingParam | None = None, + shared_session: Optional["ClientSession"] = None, + skip_responses_api_bridge: bool = False, +) -> ModelResponse | CustomStreamWrapper: + """``skip_responses_api_bridge`` is set by the Responses to chat bridge so a ``mode: responses`` deployment stays on chat.""" + api_base = kwargs.get("api_base", None) + mock_response: Final[MOCK_RESPONSE_TYPE | None] = kwargs.get("mock_response", None) + mock_tool_calls: Final = kwargs.get("mock_tool_calls", None) + mock_timeout: Final = cast(bool | None, kwargs.get("mock_timeout", None)) + force_timeout: Final = kwargs.get("force_timeout", 600) ## deprecated + logger_fn: Final = kwargs.get("logger_fn", None) + verbose: Final = kwargs.get("verbose", False) + custom_llm_provider = kwargs.get("custom_llm_provider", None) + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj", None) + id: Final = kwargs.get("id", None) + metadata: Final = kwargs.get("metadata", None) + model_info: Final = kwargs.get("model_info", None) + proxy_server_request: Final = kwargs.get("proxy_server_request", None) + provider_specific_header: Final = cast( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, + kwargs.get("provider_specific_header", None), + ) + headers = kwargs.get("headers", None) or extra_headers + + headers = {} if headers is None else dict(headers) + if extra_headers is not None: + headers.update(extra_headers) + # Inject proxy auth headers if configured + if litellm.proxy_auth is not None: + try: + proxy_headers: Final = proxy_auth_headers(litellm.proxy_auth) + headers.update(proxy_headers) + except Exception as e: + verbose_logger.warning("Failed to get proxy auth headers: %s", e) + num_retries: Final = kwargs.get( + "num_retries", None + ) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor. + max_retries = kwargs.get("max_retries", None) + cooldown_time: Final = kwargs.get("cooldown_time", None) + organization: Final = kwargs.get("organization", None) + ### VERIFY SSL ### + ssl_verify: Final = kwargs.get("ssl_verify", None) + ### CUSTOM MODEL COST ### + input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) + output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) + cost_per_second: Final = kwargs.get("cost_per_second", None) + input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) + output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) + ### CUSTOM PROMPT TEMPLATE ### + initial_prompt_value: Final = kwargs.get("initial_prompt_value", None) + roles: Final = kwargs.get("roles", None) + final_prompt_value: Final = kwargs.get("final_prompt_value", None) + bos_token: Final = kwargs.get("bos_token", None) + eos_token: Final = kwargs.get("eos_token", None) + preset_cache_key: Final = kwargs.get("preset_cache_key", None) + hf_model_name: Final = kwargs.get("hf_model_name", None) + supports_system_message: Final = kwargs.get("supports_system_message", None) + base_model: Final = kwargs.get("base_model", None) or ( + model_info.get("base_model") if isinstance(model_info, dict) else None + ) + ### DISABLE FLAGS ### + disable_add_transform_inline_image_block: Final = kwargs.get("disable_add_transform_inline_image_block", None) + ### TEXT COMPLETION CALLS ### + text_completion: Final = kwargs.get("text_completion", False) + ### ASYNC CALLS ### + acompletion: Final = kwargs.get("acompletion", False) + client: Final = kwargs.get("client", None) + ### Admin Controls ### + no_log: Final = kwargs.get("no-log", False) + + try: + if base_url is not None: + api_base = base_url + is_router_call: Final = any("model_group" in (kwargs.get(k) or ()) for k in ("metadata", "litellm_metadata")) + if is_router_call: + max_retries = 0 + elif num_retries is not None: + max_retries = num_retries + logging: Final[LiteLLMLoggingObj] = cast(LiteLLMLoggingObj, litellm_logging_obj) + if litellm.model_alias_map and model in litellm.model_alias_map: + model = litellm.model_alias_map[ + model + ] # update the model to the actual value if an alias has been passed in + model_response: Final = ModelResponse() + setattr(model_response, "usage", litellm.Usage()) + if ( + kwargs.get("azure", False) is True + ): # don't remove flag check, to remain backwards compatible for repos like Codium + custom_llm_provider = "azure" + if deployment_id is not None: # azure llms + model = deployment_id + custom_llm_provider = "azure" + _supplemental_provider_params: Final = { + **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, + "extra_headers": extra_headers, + } + requested_model: Final = model + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + litellm_params=( + GenericLiteLLMParams.model_validate(_supplemental_provider_params) + if _supplemental_provider_params + else None + ), + ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) + + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) + ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name + responses_api_model_info, model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + web_search_options=web_search_options, + api_base=api_base, + deployment_model_info=deployment_model_info, + ) + routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + drop_unsupported_reasoning: Final = ( + litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True + ) + reasoning_model_group: Final = model_group_from_metadata( + routing_kwargs.get("metadata"), routing_kwargs.get("litellm_metadata") + ) + requested_output_config: Final = as_str_mapping(non_default_params.get("output_config")) + chat_reasoning: Final = resolve_chat_reasoning( + reasoning_effort, + thinking, + requested_output_config, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=routing_kwargs.get("model_info"), + model_group=reasoning_model_group, + drop_params=drop_unsupported_reasoning, + ) + resolved_reasoning_effort: Final = chat_reasoning.effort.value + resolved_output_config: Final = chat_reasoning.output_config + provider_params: Final = ( + non_default_params + if resolved_output_config is requested_output_config + else { + **{key: value for key, value in non_default_params.items() if key != "output_config"}, + **({"output_config": resolved_output_config} if resolved_output_config is not None else {}), + } + ) + + if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): + tools = _drop_anthropic_only_tool_keys(tools=tools) + + if provider_specific_header is not None: + headers.update( + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=provider_specific_header, + custom_llm_provider=custom_llm_provider, + ) + ) + + if model_response is not None and hasattr(model_response, HIDDEN_PARAMS_ATTR): + model_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) + model_response_hidden_params["custom_llm_provider"] = custom_llm_provider + model_response_hidden_params["region_name"] = kwargs.get( + "aws_region_name", None + ) # support region-based pricing for bedrock + + ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### + if ( + (input_cost_per_token is not None and output_cost_per_token is not None) + or input_cost_per_second is not None + or output_cost_per_second is not None + or cost_per_second is not None + ): + register_custom_pricing_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) + ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### + custom_prompt_dict = {} + if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: + custom_prompt_dict = {model: {}} + if initial_prompt_value: + custom_prompt_dict[model]["initial_prompt_value"] = initial_prompt_value + if roles: + custom_prompt_dict[model]["roles"] = roles + if final_prompt_value: + custom_prompt_dict[model]["final_prompt_value"] = final_prompt_value + if bos_token: + custom_prompt_dict[model]["bos_token"] = bos_token + if eos_token: + custom_prompt_dict[model]["eos_token"] = eos_token + + messages = update_messages_with_model_file_ids( + messages=messages, + model_id=(kwargs.get("model_info") or {}).get("id", None), + model_file_id_mapping=cast( + dict[str, dict[str, str]], + kwargs.get("model_file_id_mapping") or {}, + ), + ) + + provider_config: BaseConfig | None = None + if custom_llm_provider is not None and custom_llm_provider in [provider.value for provider in LlmProviders]: + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders(custom_llm_provider), + base_model=base_model, + ) + + if provider_config is not None: + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) + history_for_route: Final = _history_without_unanswered_tool_search_calls( + translated_messages, custom_llm_provider, provider_config + ) + messages = ( # rebind-ok: completion() rewrites messages step by step for the provider + translated_messages if history_for_route is None else list(history_for_route) + ) + + if ( + supports_system_message is not None + and isinstance(supports_system_message, bool) + and supports_system_message is False + ): + messages = map_system_message_pt(messages=messages) + + if dynamic_api_key is not None: + api_key = dynamic_api_key + # check if user passed in any of the OpenAI optional params + bridges_to_responses_api: Final = not skip_responses_api_bridge and _bridges_to_responses_api( + responses_api_model_info, model, custom_llm_provider, model_info, api_base + ) + allowed_openai_params: Final[list[str] | None] = ( + [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] + if bridges_to_responses_api + or reasoning_effort_needs_allowing( + chat_reasoning.effort, + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, + ) + else kwargs.get("allowed_openai_params") + ) + optional_param_args: Final = { + "functions": functions, + "function_call": function_call, + "temperature": temperature, + "top_p": top_p, + "n": n, + "stream": stream, + "stream_options": stream_options, + "stop": stop, + "max_tokens": max_tokens, + "max_completion_tokens": max_completion_tokens, + "modalities": modalities, + "prediction": prediction, + "audio": audio, + "presence_penalty": presence_penalty, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "user": user, + # params to identify the model + "model": model, + "custom_llm_provider": custom_llm_provider, + "response_format": response_format, + "seed": seed, + "tools": tools, + "tool_choice": tool_choice, + "max_retries": max_retries, + "logprobs": logprobs, + "top_logprobs": top_logprobs, + "api_version": api_version, + "parallel_tool_calls": parallel_tool_calls, + "messages": messages, + "reasoning_effort": resolved_reasoning_effort, + "verbosity": verbosity, + "thinking": chat_reasoning.thinking, + "web_search_options": web_search_options, + "include_server_side_tool_invocations": ( + include_server_side_tool_invocations + if include_server_side_tool_invocations is not None + else kwargs.get("include_server_side_tool_invocations") + ), + "safety_identifier": safety_identifier, + "service_tier": service_tier, + "store": store, + "prompt_cache_key": prompt_cache_key, + "prompt_cache_retention": prompt_cache_retention, + "allowed_openai_params": allowed_openai_params, + "base_model": base_model, + } + provider_reasoning_effort: Final = reasoning_effort_with_summary_alias( + resolved_reasoning_effort, + provider_params, + accepts_summary=provider_config is not None + and provider_config.accepts_reasoning_summary_in_reasoning_effort, + ) + + mapping_args: Final = MappingProxyType({**optional_param_args, "reasoning_effort": provider_reasoning_effort}) + mapping_special_params: Final = MappingProxyType(dict(provider_params)) + + def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: + return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) + + validated_model_info: Final = ( + _MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None + ) + deployment_base_model: Final = base_model if isinstance(base_model, str) else None + optional_params = optional_params_for_model_info(validated_model_info) + fakes_stream: Final = partial( + _completion_fakes_stream, + model, + custom_llm_provider, + provider_config, + base_model=deployment_base_model, + bridges_to_responses_api=bridges_to_responses_api is True, + ) + resolve_timeout: Final = partial( + CompletionTimeout.resolve, + timeout, + kwargs, + custom_llm_provider, + global_timeout=get_configured_request_timeout(), + supports_httpx_timeout=supports_httpx_timeout, + ) + streams: Final = stream is True + + def timeout_for_optional_params(attempt_params: Mapping[str, object]) -> float | httpx.Timeout: + if not streams: + return resolve_timeout(stream_kind=StreamKind.NOT_STREAMED) + return resolve_timeout(stream_kind=StreamKind.FAKED if fakes_stream(attempt_params) else StreamKind.LIVE) + + ### TIMEOUT LOGIC ### + timeout = timeout_for_optional_params(optional_params) + logging.record_applied_timeout(timeout) + processed_non_default_params: Final = pre_process_non_default_params( + model=model, + passed_params=optional_param_args, + special_params=provider_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=kwargs.get("additional_drop_params"), + remove_sensitive_keys=True, + add_provider_specific_params=True, + provider_config=provider_config, + ) + + if litellm.add_function_to_prompt and optional_params.get( + "functions_unsupported_model", None + ): # if user opts to add it to prompt, when API doesn't support function calling + functions_unsupported_model: Final[list[object]] = TypeAdapter(list[object]).validate_python( + optional_params.pop("functions_unsupported_model") + ) + messages = function_call_prompt( + messages=messages, + functions=functions_unsupported_model, + model=model, + llm_provider=custom_llm_provider, + drop_params=drop_unsupported_reasoning, + ) + + ensure_alternating_roles: Final = cast( # cast-ok: read from untyped kwargs + bool | None, kwargs.get("ensure_alternating_roles", None) + ) + user_continue_message: Final = cast( # cast-ok: read from untyped kwargs + ChatCompletionUserMessage | None, kwargs.get("user_continue_message", None) + ) + assistant_continue_message: Final = cast( # cast-ok: read from untyped kwargs + ChatCompletionAssistantMessage | None, kwargs.get("assistant_continue_message", None) + ) + readable_messages: Final = _messages_readable_by_model( + messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List + model=model, + custom_llm_provider=custom_llm_provider, + model_info=validated_model_info, + base_model=deployment_base_model, + drop_params=drop_unsupported_reasoning, + ensure_alternating_roles=ensure_alternating_roles, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + messages = ( # rebind-ok: completion() rewrites messages step by step + messages if readable_messages is None else list(readable_messages) + ) + + # For logging - save the values of the litellm-specific params passed in + requested_litellm_params: Final = get_litellm_params( + acompletion=acompletion, + api_key=api_key, + force_timeout=force_timeout, + logger_fn=logger_fn, + verbose=verbose, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + litellm_call_id=kwargs.get("litellm_call_id", None), + model_alias_map=litellm.model_alias_map, + completion_call_id=id, + metadata=metadata, + model_info=model_info, + proxy_server_request=proxy_server_request, + preset_cache_key=preset_cache_key, + no_log=no_log, + cost_per_second=cost_per_second, + input_cost_per_second=input_cost_per_second, + input_cost_per_token=input_cost_per_token, + output_cost_per_second=output_cost_per_second, + output_cost_per_token=output_cost_per_token, + cooldown_time=cooldown_time, + text_completion=kwargs.get("text_completion"), + azure_ad_token_provider=kwargs.get("azure_ad_token_provider"), + user_continue_message=kwargs.get("user_continue_message"), + base_model=base_model, + litellm_trace_id=kwargs.get("litellm_trace_id"), + litellm_session_id=kwargs.get("litellm_session_id"), + hf_model_name=hf_model_name, + custom_prompt_dict=custom_prompt_dict, + litellm_metadata=kwargs.get("litellm_metadata"), + disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, + drop_params=kwargs.get("drop_params"), + prompt_id=prompt_id, + prompt_variables=prompt_variables, + ssl_verify=ssl_verify, + merge_reasoning_content_in_choices=kwargs.get("merge_reasoning_content_in_choices", None), + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + api_version=api_version, + azure_ad_token=kwargs.get("azure_ad_token"), + tenant_id=kwargs.get("tenant_id"), + client_id=kwargs.get("client_id"), + client_secret=kwargs.get("client_secret"), + azure_username=kwargs.get("azure_username"), + azure_password=kwargs.get("azure_password"), + azure_scope=kwargs.get("azure_scope"), + max_retries=max_retries, + timeout=timeout, + litellm_request_debug=kwargs.get("litellm_request_debug", False), + tpm=kwargs.get("tpm"), + rpm=kwargs.get("rpm"), + use_xai_oauth=kwargs.get("use_xai_oauth", False), + chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"), + gigachat_scope=kwargs.get("gigachat_scope"), + gigachat_auth_url=kwargs.get("gigachat_auth_url"), + gigachat_access_token=kwargs.get("gigachat_access_token"), + **{ + key: kwargs[key] + for key in ( + *AWS_CREDENTIAL_KWARGS_KEYS, + *ANTHROPIC_WIF_KWARGS_KEYS, + *OPENAI_WIF_KWARGS_KEYS, + *OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, + "github_copilot_auth_type", + "github_copilot_host", + "github_copilot_user_session", + PROVIDER_AFFINITY_HEADER_KWARG_KEY, + "fireworks_forward_user_id", + "responses_api_version", + ) + if key in kwargs + }, + ) + litellm_params: Final = with_control_options(requested_litellm_params, control_options) + affinity_api_base: Final = ( + str(client.base_url) + if isinstance(client, openai.OpenAI | openai.AsyncOpenAI) + else get_openai_credentials(api_base=api_base).api_base + if custom_llm_provider == "openai" + else api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) + if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None: + try: + headers = add_provider_affinity_header( + headers=headers or litellm.headers or MappingProxyType({}), + litellm_params=MappingProxyType( + { + "provider_affinity_header": litellm_params.get("provider_affinity_header"), + "litellm_session_id": kwargs.get("litellm_session_id"), + "session_id": kwargs.get("session_id"), + "metadata": metadata, + "litellm_metadata": kwargs.get("litellm_metadata"), + } + ), + default_header=default_affinity_header, + ) + except ValueError as affinity_error: + raise litellm.BadRequestError( + message=str(affinity_error), model=model, llm_provider=custom_llm_provider + ) from affinity_error + logging.update_environment_variables( + model=model, + user=user, + optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. + litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, + model_id=optional_params.get("model_id"), + ) + if mock_response is not None or mock_tool_calls or mock_timeout: + kwargs.pop("mock_timeout", None) # remove for any fallbacks triggered + return mock_completion( + model, + messages, + stream=stream, + n=n, + mock_response=mock_response, + mock_tool_calls=mock_tool_calls, + logging=logging, + acompletion=acompletion, + mock_delay=kwargs.get("mock_delay", None), + custom_llm_provider=custom_llm_provider, + mock_timeout=mock_timeout, + timeout=timeout, + prompt_tokens=admission_input_tokens( + cast(Mapping[str, object], kwargs) # cast-ok: completion's **kwargs is untyped + ), + ) + + ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map + # Only run the second bridge check if the first one didn't already + # detect responses mode (e.g. via the "responses/" prefix). The second + # check handles cases like gpt-5.4+ with tools+reasoning_effort or + # reasoningSummary/reasoning_summary without tools (AI SDK) that the first + # (early) check doesn't cover. + _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params) + if responses_api_model_info.get("mode") != "responses": + responses_api_model_info, model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + web_search_options=web_search_options, + tools=tools, + reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")), + reasoning_summary=_reasoning_summary_for_bridge, + api_base=api_base, + deployment_model_info=str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ), + ) + + # Use base_model (the true underlying model) for Azure model-type + # detection when the deployment name differs from the model name. + _azure_detection_model: Final = base_model or model + + if not skip_responses_api_bridge and _bridges_to_responses_api( + responses_api_model_info, model, custom_llm_provider, model_info, api_base + ): + from litellm.completion_extras import responses_api_bridge + + optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) + + if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict): + from litellm.responses.utils import openai_bridge_reasoning + + optional_params["reasoning_effort"] = openai_bridge_reasoning( + requested=resolved_reasoning_effort, + effort_kept="reasoning_effort" in optional_params, + summary_alias=rs_val, + model=model, + drop_params=drop_unsupported_reasoning, + additional_drop_params=routing_kwargs.get("additional_drop_params"), + ) + elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: + optional_params["reasoning_effort"] = resolved_reasoning_effort + elif rs_val is not None: + eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) + if isinstance(eff, dict): + optional_params["reasoning_effort"] = {**eff, "summary": rs_val} + elif eff is not None: + optional_params["reasoning_effort"] = { + "effort": eff, + "summary": rs_val, + } + else: + optional_params["reasoning_effort"] = {"summary": rs_val} + + return responses_api_bridge.completion( # pyright: ignore[reportReturnType] # bridge returns a coroutine on the acompletion path; awaited by the async caller + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, # pass AsyncOpenAI, OpenAI client + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + cache=kwargs.get("cache"), + ) + elif (custom_llm_provider == "openai" and OpenAIGPT5Config.is_model_gpt_5_model(model)) or ( + custom_llm_provider == "azure" + and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(_azure_detection_model) + ): + optional_params, _ = strip_reasoning_summary_aliases_from_optional_params(optional_params) + + _dispatch_ctx: Final = CompletionDispatchContext( + _azure_detection_model=_azure_detection_model, + acompletion=acompletion, + api_base=api_base, + api_key=api_key, + api_version=api_version, + client=client, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + extra_headers=extra_headers, + headers=headers, + hf_model_name=hf_model_name, + kwargs=kwargs, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging=logging, + max_retries=max_retries, + max_tokens=max_tokens, + messages=messages, + metadata=metadata, + model=model, + model_response=model_response, + optional_params=optional_params, + optional_params_for_model_info=optional_params_for_model_info, + organization=organization, + provider_config=provider_config, + request_params=MappingProxyType( + OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **provider_params}) + ), + shared_session=shared_session, + stream=stream, + temperature=temperature, + text_completion=text_completion, + timeout=timeout, + timeout_for_optional_params=timeout_for_optional_params, + top_p=top_p, + ) + if custom_llm_provider == "azure": + # azure configs + ## check dynamic params ## + response = complete_azure(_dispatch_ctx) + elif custom_llm_provider == "azure_text": + # azure configs + response = complete_azure_text(_dispatch_ctx) + elif custom_llm_provider == "deepseek": + ## COMPLETION CALL + + response = complete_deepseek(_dispatch_ctx) + + elif custom_llm_provider == "azure_ai": + response = complete_azure_ai(_dispatch_ctx) + elif ( + custom_llm_provider == "text-completion-openai" + or "ft:babbage-002" in model + or "ft:davinci-002" in model # support for finetuned completion models + or custom_llm_provider in litellm.openai_text_completion_compatible_providers + and kwargs.get("text_completion") is True + ): + response = complete_text_completion_openai(_dispatch_ctx) + elif custom_llm_provider == "fireworks_ai": + ## COMPLETION CALL + response = complete_fireworks_ai(_dispatch_ctx) + elif custom_llm_provider == "together_ai": + response = complete_together_ai(_dispatch_ctx) + elif custom_llm_provider == "heroku": + response = complete_heroku(_dispatch_ctx) + + elif custom_llm_provider == "ragflow": + ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths + response = complete_ragflow(_dispatch_ctx) + elif custom_llm_provider == "scaledown": + response = complete_scaledown(_dispatch_ctx) # rebind-ok: shared provider return + elif custom_llm_provider == "xai": + ## COMPLETION CALL + response = complete_xai(_dispatch_ctx) + elif custom_llm_provider == "groq": + response = complete_groq(_dispatch_ctx) + elif custom_llm_provider == "opencode_go": + response = complete_opencode_go(_dispatch_ctx) + elif custom_llm_provider == "bedrock_mantle": + response = complete_bedrock_mantle(_dispatch_ctx) + elif custom_llm_provider == "a2a": + # A2A (Agent-to-Agent) Protocol + # Resolve agent configuration from registry if model format is "a2a/" + response = complete_a2a(_dispatch_ctx) + elif custom_llm_provider == "gigachat": + # GigaChat - Sber AI's LLM (Russia) + response = complete_gigachat(_dispatch_ctx) + + elif custom_llm_provider == "sap": + response = complete_sap(_dispatch_ctx) + elif custom_llm_provider == "aiohttp_openai": + # NEW aiohttp provider for 10-100x higher RPS + response = complete_aiohttp_openai(_dispatch_ctx) + elif custom_llm_provider == "cometapi": + response = complete_cometapi(_dispatch_ctx) + elif custom_llm_provider == "minimax": + response = complete_minimax(_dispatch_ctx) + elif custom_llm_provider == "hosted_vllm": + response = complete_hosted_vllm(_dispatch_ctx) + elif custom_llm_provider == "edenai": + response = complete_edenai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch + elif custom_llm_provider == "fal_ai": + response = complete_fal_ai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch + elif ( + # A known OpenAI model name only decides the route when nothing else + # resolved a provider. get_llm_provider() already maps these names to + # "openai", so a different value here was asked for explicitly (or came + # from a register_model entry), and the provider config built for it + # would be handed to the OpenAI handler. + (model in litellm.open_ai_chat_completion_models and custom_llm_provider in (None, "openai")) + or custom_llm_provider == "custom_openai" + or custom_llm_provider == "deepinfra" + or custom_llm_provider == "perplexity" + or custom_llm_provider == "nvidia_nim" + or custom_llm_provider == "cerebras" + or custom_llm_provider == "baseten" + or custom_llm_provider == "sambanova" + or custom_llm_provider == "volcengine" + or custom_llm_provider == "anyscale" + or custom_llm_provider == "openai" + or custom_llm_provider == "nebius" + or custom_llm_provider == "wandb" + or custom_llm_provider == "clarifai" + or custom_llm_provider == "bigmodel" + or custom_llm_provider in litellm.openai_compatible_providers + or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers + or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo + ): # allow user to make an openai call with a custom base + # note: if a user sets a custom base - we should ensure this works + # allow for the setting of dynamic and stateful api-bases + response = complete_custom_openai(_dispatch_ctx) + + elif custom_llm_provider == "mistral": + response = complete_mistral(_dispatch_ctx) + elif "replicate" in model or custom_llm_provider == "replicate" or model in litellm.replicate_models: + # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") + response = complete_replicate(_dispatch_ctx) + elif "clarifai" in model or custom_llm_provider == "clarifai" or model in litellm.clarifai_models: + pass # Deprecated - handled in the openai compatible provider section above + elif custom_llm_provider == "anthropic_text": + response = complete_anthropic_text(_dispatch_ctx) + elif custom_llm_provider == "anthropic": + response = complete_anthropic(_dispatch_ctx) + elif custom_llm_provider == "nlp_cloud": + response = complete_nlp_cloud(_dispatch_ctx) + elif custom_llm_provider == "aleph_alpha": + response = complete_aleph_alpha(_dispatch_ctx) + elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": + response = complete_cohere_chat(_dispatch_ctx) + elif custom_llm_provider == "maritalk": + response = complete_maritalk(_dispatch_ctx) + elif custom_llm_provider == "amazon_nova": + response = complete_amazon_nova(_dispatch_ctx) + elif custom_llm_provider == "huggingface": + response = complete_huggingface(_dispatch_ctx) + elif custom_llm_provider == "oci": + response = complete_oci(_dispatch_ctx) + elif custom_llm_provider == "compactifai": + response = complete_compactifai(_dispatch_ctx) + elif custom_llm_provider == "oobabooga": + response = complete_oobabooga(_dispatch_ctx) + elif custom_llm_provider == "databricks": + response = complete_databricks(_dispatch_ctx) + + elif custom_llm_provider == "datarobot": + response = complete_datarobot(_dispatch_ctx) + elif custom_llm_provider == "openrouter": + response = complete_openrouter(_dispatch_ctx) + elif custom_llm_provider == "nadir": + response = complete_nadir(_dispatch_ctx) # rebind-ok: mirrors sibling provider branches + elif custom_llm_provider == "vercel_ai_gateway": + response = complete_vercel_ai_gateway(_dispatch_ctx) + elif custom_llm_provider == "palm": + raise ValueError( + "Palm was decommissioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" + ) + elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini": + response = complete_vertex_ai_beta(_dispatch_ctx) + + elif custom_llm_provider == "vertex_ai": + response = complete_vertex_ai(_dispatch_ctx) + elif custom_llm_provider == "predibase": + response = complete_predibase(_dispatch_ctx) + elif custom_llm_provider == "text-completion-codestral": + response = complete_text_completion_codestral(_dispatch_ctx) + elif custom_llm_provider == "text-completion-inception": + response = complete_text_completion_inception(_dispatch_ctx) + elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): + # boto3 reads keys from .env + # sagemaker_chat: HF Messages API endpoints + # sagemaker_nova: Nova models on SageMaker (OpenAI-compatible) + response = complete_sagemaker_chat(_dispatch_ctx) + elif custom_llm_provider == "sagemaker": + # boto3 reads keys from .env + response = complete_sagemaker(_dispatch_ctx) + elif custom_llm_provider == "bedrock": + # boto3 reads keys from .env + response = complete_bedrock(_dispatch_ctx) + elif custom_llm_provider == "watsonx": + response = complete_watsonx(_dispatch_ctx) + elif custom_llm_provider == "watsonx_text": + response = complete_watsonx_text(_dispatch_ctx) + elif custom_llm_provider == "vllm": + response = complete_vllm(_dispatch_ctx) + elif custom_llm_provider == "ollama": + response = complete_ollama(_dispatch_ctx) + + elif custom_llm_provider == "ollama_chat": + response = complete_ollama_chat(_dispatch_ctx) + + elif custom_llm_provider == "triton": + response = complete_triton(_dispatch_ctx) + elif custom_llm_provider == "cloudflare": + response = complete_cloudflare(_dispatch_ctx) + + elif custom_llm_provider == "petals" or model in litellm.petals_models: + response = complete_petals(_dispatch_ctx) + elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: + response = complete_snowflake(_dispatch_ctx) + elif custom_llm_provider == "gradient_ai": + response = complete_gradient_ai(_dispatch_ctx) + + elif custom_llm_provider == "gdc": + response = complete_gdc(_dispatch_ctx) + elif custom_llm_provider == "bytez": + response = complete_bytez(_dispatch_ctx) + elif custom_llm_provider == "lemonade": + response = complete_lemonade(_dispatch_ctx) + + elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models: + response = complete_ovhcloud(_dispatch_ctx) + + elif custom_llm_provider == "custom": + response = complete_custom(_dispatch_ctx) + + elif custom_llm_provider in litellm._custom_providers: # Assume custom LLM provider + # Get the Custom Handler + response = complete_custom_providers(_dispatch_ctx) + + elif custom_llm_provider == "langgraph": + # LangGraph - Agent Runtime Provider + response = complete_langgraph(_dispatch_ctx) + + elif custom_llm_provider == "microsoft_365_copilot": + response = complete_microsoft_365_copilot(_dispatch_ctx) # rebind-ok: shared provider return + + elif custom_llm_provider == "langflow": + # LangFlow - Visual AI Agent Platform + response = complete_langflow(_dispatch_ctx) + + elif custom_llm_provider == "databricks_agent": + response = complete_databricks_agent(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch + + else: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) + return response + except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception completion() raised + ## Map to OpenAI Exception + raise exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={"messages": messages}, + extra_kwargs=kwargs, + ) diff --git a/litellm/native/chat_mock.py b/litellm/native/chat_mock.py new file mode 100644 index 00000000000..74da2c0d1a6 --- /dev/null +++ b/litellm/native/chat_mock.py @@ -0,0 +1,281 @@ +import asyncio +import time +from typing import ( + Final, + cast, # noqa: TID251 # casts moved verbatim from litellm/main.py +) + +import httpx +import openai + +import litellm +from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT +from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming +from litellm.native.request_support import admitted_prompt_token_counter +from litellm.types.utils import ChatCompletionMessageToolCall, ModelResponseStream +from litellm.utils import ( + CustomStreamWrapper, + ModelResponse, + Usage, + async_mock_completion_streaming_obj, + mock_completion_streaming_obj, +) + +MOCK_RESPONSE_TYPE = str | Exception | dict | ModelResponse | ModelResponseStream + + +def handle_mock_potential_exceptions( + mock_response: str | Exception, + model: str, + custom_llm_provider: str | None = None, +) -> None: + if isinstance(mock_response, Exception): + if isinstance(mock_response, openai.APIError): + raise mock_response + raise litellm.MockException( + status_code=getattr(mock_response, "status_code", 500), + message=getattr(mock_response, "text", str(mock_response)), + llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), + model=model, + request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), + ) + elif isinstance(mock_response, str) and mock_response == "litellm.RateLimitError": + raise litellm.RateLimitError( + message="this is a mock rate limit error", + llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), + model=model, + ) + elif isinstance(mock_response, str) and mock_response == "litellm.ContextWindowExceededError": + raise litellm.ContextWindowExceededError( + message="this is a mock context window exceeded error", + llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), + model=model, + ) + elif isinstance(mock_response, str) and mock_response == "litellm.InternalServerError": + raise litellm.InternalServerError( + message="this is a mock internal server error", + llm_provider=getattr(mock_response, "llm_provider", custom_llm_provider or "openai"), + model=model, + ) + elif isinstance(mock_response, str) and mock_response.startswith("Exception: content_filter_policy"): + raise litellm.MockException( + status_code=400, + message=mock_response, + llm_provider="azure", + model=model, + request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), + ) + + +def handle_mock_timeout( + mock_timeout: bool | None, + timeout: float | str | httpx.Timeout | None, + model: str, +) -> None: + if mock_timeout is True and timeout is not None: + _sleep_for_timeout(timeout) + raise litellm.Timeout( + message="This is a mock timeout error", + llm_provider="openai", + model=model, + ) + + +async def handle_mock_timeout_async( + mock_timeout: bool | None, + timeout: float | str | httpx.Timeout | None, + model: str, +) -> None: + if mock_timeout is True and timeout is not None: + await _sleep_for_timeout_async(timeout) + raise litellm.Timeout( + message="This is a mock timeout error", + llm_provider="openai", + model=model, + ) + + +def _sleep_for_timeout(timeout: float | str | httpx.Timeout): + if isinstance(timeout, float): + time.sleep(timeout) + elif isinstance(timeout, str): + time.sleep(float(timeout)) + elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None: + time.sleep(timeout.connect) + + +async def _sleep_for_timeout_async(timeout: float | str | httpx.Timeout): + if isinstance(timeout, float): + await asyncio.sleep(timeout) + elif isinstance(timeout, str): + await asyncio.sleep(float(timeout)) + elif isinstance(timeout, httpx.Timeout) and timeout.connect is not None: + await asyncio.sleep(timeout.connect) + + +def mock_completion( + model: str, + messages: list, + stream: bool | None = False, + n: int | None = None, + mock_response: MOCK_RESPONSE_TYPE | None = "This is a mock request", + mock_tool_calls: list | None = None, + mock_timeout: bool | None = False, + logging=None, + custom_llm_provider=None, + timeout: float | str | httpx.Timeout | None = None, + prompt_tokens: int | None = None, + **kwargs, +): + """ + Generate a mock completion response for testing or debugging purposes. + + This is a helper function that simulates the response structure of the OpenAI completion API. + + Parameters: + model (str): The name of the language model for which the mock response is generated. + messages (List): A list of message objects representing the conversation context. + stream (bool, optional): If True, returns a mock streaming response (default is False). + mock_response (str, optional): The content of the mock response (default is "This is a mock request"). + mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False). + timeout (float, optional): The timeout value to use for the mock response (default is None). + **kwargs: Additional keyword arguments that can be used but are not required. + + Returns: + litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response. + + Raises: + Exception: If an error occurs during the generation of the mock completion response. + Note: + - This function is intended for testing or debugging purposes to generate mock completion responses. + - If 'stream' is True, it returns a response that mimics the behavior of a streaming completion. + """ + try: + is_acompletion: Final = kwargs.get("acompletion") or False + if mock_response is None: + mock_response = "This is a mock request" + + handle_mock_timeout(mock_timeout=mock_timeout, timeout=timeout, model=model) + + ## LOGGING + if logging is not None: + logging.pre_call( + input=messages, + api_key="mock-key", + ) + + if isinstance(mock_response, str) or isinstance(mock_response, Exception): + handle_mock_potential_exceptions( + mock_response=mock_response, + model=model, + custom_llm_provider=custom_llm_provider, + ) + + mock_response = cast( + str | dict | ModelResponse | ModelResponseStream, mock_response + ) # after this point, mock_response is a string, dict, ModelResponse, or ModelResponseStream + if isinstance(mock_response, str) and mock_response.startswith("Exception: mock_streaming_error"): + mock_response = litellm.MockException( + message="This is a mock error raised mid-stream", + llm_provider="anthropic", + model=model, + status_code=529, + ) + time_delay: Final = kwargs.get("mock_delay", None) + if time_delay is not None and not is_acompletion: + time.sleep(time_delay) + + if isinstance(mock_response, dict): + return ModelResponse(**mock_response) + + if isinstance(mock_response, ModelResponse): + if not stream: + return mock_response + # convert to ModelResponseStream + mock_response = convert_model_response_to_streaming(mock_response) + + model_response: ModelResponse | ModelResponseStream = ModelResponse() + + if stream is True: + model_response = ModelResponseStream() + count_prompt_tokens: Final = admitted_prompt_token_counter(prompt_tokens) + # don't try to access stream object, + if kwargs.get("acompletion", False) is True: + return CustomStreamWrapper( + completion_stream=async_mock_completion_streaming_obj( + model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging, + count_prompt_tokens=count_prompt_tokens, + ) + return CustomStreamWrapper( + completion_stream=mock_completion_streaming_obj( + model_response, mock_response=mock_response, model=model, n=n, prompt_tokens=prompt_tokens + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging, + count_prompt_tokens=count_prompt_tokens, + ) + if isinstance(mock_response, litellm.MockException): + raise mock_response + # At this point, mock_response must be a string (all other types have been handled or returned early) + mock_response = cast(str, mock_response) + + if n is None: + model_response.choices[0].message.content = mock_response + else: + _all_choices: Final = [] + for i in range(n): + _choice = litellm.utils.Choices( + index=i, + message=litellm.utils.Message(content=mock_response, role="assistant"), + ) + _all_choices.append(_choice) + model_response.choices = _all_choices + model_response.created = int(time.time()) + model_response.model = model + + if mock_tool_calls: + model_response.choices[0].message.tool_calls = [ + ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls + ] + + usage_prompt_tokens: Final = ( + prompt_tokens if prompt_tokens is not None else DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + ) + setattr( + model_response, + "usage", + Usage( + prompt_tokens=usage_prompt_tokens, + completion_tokens=DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + total_tokens=usage_prompt_tokens + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + ), + ) + + if custom_llm_provider is not None: + model_response.hidden_params["custom_llm_provider"] = custom_llm_provider + else: + try: + _, inferred_provider, _, _ = litellm.utils.get_llm_provider(model=model) + model_response.hidden_params["custom_llm_provider"] = inferred_provider + except Exception: + # dont let setting a hidden param block a mock_respose + pass + + if logging is not None: + logging.post_call( + input=messages, + api_key="my-secret-key", + original_response="my-original-response", + ) + + return model_response + + except Exception as e: + if isinstance(e, openai.APIError): + raise e + raise Exception(f"Mock completion response failed - {e}") diff --git a/litellm/native/chat_plan.py b/litellm/native/chat_plan.py new file mode 100644 index 00000000000..b990edfaf7b --- /dev/null +++ b/litellm/native/chat_plan.py @@ -0,0 +1,283 @@ +import asyncio +import contextvars +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from functools import partial +from types import MappingProxyType +from typing import ( + Final, + cast, # noqa: TID251 # casts moved verbatim from litellm/main.py +) + +from pydantic import TypeAdapter + +import litellm +from litellm.litellm_core_utils.chat_completion_agentic_loop import maybe_run_chat_completion_agentic_loop +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.prompt_templates.common_utils import get_completion_messages +from litellm.litellm_core_utils.rules import Rules +from litellm.native.chat import OPTIONAL_PARAMS_ADAPTER, execute_chat, resolve_dispatched_chat_response +from litellm.native.request_support import resolve_control_options +from litellm.types.litellm_params import ControlOptions +from litellm.types.llms.anthropic import AnthropicThinkingParam +from litellm.utils import ( + CustomStreamWrapper, + ModelResponse, + exception_type, + get_llm_provider, + get_non_default_completion_params, + post_call_processing, + validate_and_fix_openai_messages, + validate_and_fix_openai_tools, + validate_and_fix_thinking_param, + validate_chat_completion_tool_choice, + validate_openai_optional_params, +) + +_PLANNED_CHAT_ARGS: Final = frozenset( + ("kwargs", "non_default_params", "control_options", "prompt_id", "prompt_variables") +) + + +_EXECUTE_CHAT_CODE: Final = execute_chat.__code__ + + +_EXECUTE_CHAT_PARAMS: Final = ( + frozenset(_EXECUTE_CHAT_CODE.co_varnames[: _EXECUTE_CHAT_CODE.co_kwonlyargcount]) - _PLANNED_CHAT_ARGS +) + + +_SKIP_RESPONSES_API_BRIDGE_KWARG: Final = "_skip_responses_api_bridge" + + +_CHAT_SURFACE_KWARGS: Final = ( + "fallbacks", + "model_list", + "prompt_id", + "litellm_system_prompt", + "ensure_alternating_roles", + "user_continue_message", + "assistant_continue_message", + "mock_delay", + "mock_timeout", +) + + +_TEXT_COMPLETION_PROVIDERS: Final = frozenset( + ("text-completion-openai", "text-completion-codestral", "text-completion-inception") +) + + +@dataclass(frozen=True, slots=True) +class PlannedChat: + model: str + custom_llm_provider: str + named: Mapping[str, object] + kwargs: Mapping[str, object] + non_default_params: Mapping[str, object] + control_options: ControlOptions + + +def _optional_str(value: object) -> str | None: + return value if isinstance(value, str) else None + + +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +def _object_list(value: object) -> list[object] | None: # mutable-ok: feeds validate_and_fix_openai_* which take a list + return _OBJECT_LIST.validate_python(value) if isinstance(value, list) else None + + +def _tool_choice(value: object) -> str | dict[str, object] | None: # mutable-ok: its validator takes a dict + if isinstance(value, str): + return value + return OPTIONAL_PARAMS_ADAPTER.validate_python(value) if isinstance(value, dict) else None + + +def _stop(value: object) -> str | list[str] | None: # mutable-ok: validate_openai_optional_params takes a list + if isinstance(value, list): + return [item for item in _OBJECT_LIST.validate_python(value) if isinstance(item, str)] + return value if isinstance(value, str) else None + + +def _thinking(value: object) -> AnthropicThinkingParam | bool | None: + if isinstance(value, dict): + return cast(AnthropicThinkingParam, value) # cast-ok: the caller's thinking dict, normalized by the validator + return value if isinstance(value, bool) else None + + +def _chat_entry_points_wrapped() -> bool: + """Tracers such as ddtrace wrap `litellm.completion`/`acompletion`, so bridged calls must go through them.""" + from litellm.chat_completions import dispatch + + return litellm.completion is not dispatch.completion or litellm.acompletion is not dispatch.acompletion + + +def _reads_chat_cache(request: Mapping[str, object]) -> bool: + """The chat response cache is read and written by the `acompletion` surface, keyed on the chat request.""" + return litellm.cache is not None or request.get("caching") is True + + +def _uses_chat_surface(request: Mapping[str, object]) -> bool: + from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + from litellm.types.llms.openai import ToolParam + + tools: Final = request.get("tools") + return ( + litellm.model_fallbacks is not None + or _reads_chat_cache(request) + or _chat_entry_points_wrapped() + or any(request.get(key) is not None for key in _CHAT_SURFACE_KWARGS) + or ( + isinstance(tools, list) + and LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway( + tools=cast(Iterable[ToolParam], tools) # cast-ok: OpenAI-format tools from the caller's request + ) + ) + ) + + +def plan_chat(request: Mapping[str, object], logging_obj: LiteLLMLoggingObj) -> PlannedChat | None: + model: Final = request.get("model") + if not isinstance(model, str) or _uses_chat_surface(request): + return None + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + custom_llm_provider: Final = _optional_str(request.get("custom_llm_provider")) + api_base: Final = _optional_str(request.get("api_base")) + _, resolved_provider, _, _ = get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + ) + if resolved_provider in _TEXT_COMPLETION_PROVIDERS: + return None + kwargs: Final = { + key: value + for key, value in request.items() + if key not in _EXECUTE_CHAT_PARAMS and key != _SKIP_RESPONSES_API_BRIDGE_KWARG + } + tools: Final = validate_and_fix_openai_tools(tools=_object_list(request.get("tools"))) + messages: Final = get_completion_messages( + messages=validate_and_fix_openai_messages(messages=_object_list(request.get("messages")) or []), + ensure_alternating_roles=False, + user_continue_message=None, + assistant_continue_message=None, + ) + non_default_params: Final = get_non_default_completion_params(kwargs=kwargs) + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=non_default_params, + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + tools=tools, + enable_prompt_caching=request.get("enable_prompt_caching") is True, + api_base=api_base, + ) + if logging_obj.should_run_prompt_management_hooks(non_default_params=non_default_params, tools=tools): + return None + named: Final = {key: value for key, value in request.items() if key in _EXECUTE_CHAT_PARAMS} + tool_choice: Final = validate_chat_completion_tool_choice( + tool_choice=_tool_choice(request.get("tool_choice")), model=model + ) + return PlannedChat( + model=model, + custom_llm_provider=resolved_provider, + named=MappingProxyType( + { + **named, + "messages": messages, + "tools": tools, + "tool_choice": tool_choice, + "stop": validate_openai_optional_params(stop=_stop(request.get("stop"))), + "thinking": validate_and_fix_thinking_param(thinking=_thinking(request.get("thinking"))), + "skip_responses_api_bridge": request.get(_SKIP_RESPONSES_API_BRIDGE_KWARG) is True, + } + ), + kwargs=MappingProxyType(kwargs), + non_default_params=MappingProxyType(non_default_params), + control_options=resolve_control_options(kwargs, model), + ) + + +def _run_planned_chat(plan: PlannedChat, is_async: bool) -> ModelResponse | CustomStreamWrapper: + return execute_chat( + **plan.named, # pyright: ignore[reportArgumentType] # named holds the caller's validated OpenAI-format params + kwargs={**plan.kwargs, "acompletion": is_async}, + non_default_params=dict(plan.non_default_params), + control_options=plan.control_options, + prompt_id=None, + prompt_variables=None, + ) + + +def _post_call_checked( + plan: PlannedChat, response: ModelResponse | CustomStreamWrapper, is_async: bool +) -> ModelResponse | CustomStreamWrapper: + """Runs the post-call rules and JSON-schema validation the ``completion`` client wrapper runs on its result.""" + post_call_processing( + original_response=response, + model=plan.model, + optional_params={**plan.kwargs, **plan.named}, + original_function=litellm.acompletion if is_async else litellm.completion, + rules_obj=Rules(), + ) + return response + + +def execute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper: + return _post_call_checked(plan, _run_planned_chat(plan, is_async=False), is_async=False) + + +async def _settled_chat_response( + plan: PlannedChat, loop: asyncio.AbstractEventLoop +) -> ModelResponse | CustomStreamWrapper: + pending: Final[object] = await loop.run_in_executor( + None, partial(contextvars.copy_context().run, partial(_run_planned_chat, plan, True)) + ) + if asyncio.iscoroutine(pending): + return await resolve_dispatched_chat_response(pending) + if isinstance(pending, (ModelResponse, CustomStreamWrapper)): + return pending + raise TypeError(f"chat provider returned {type(pending).__name__}, expected a ModelResponse or a stream") + + +async def _with_agentic_loop(plan: PlannedChat, response: ModelResponse) -> ModelResponse | CustomStreamWrapper: + looped: Final = await maybe_run_chat_completion_agentic_loop( + response=response, + model=plan.model, + messages=_object_list(plan.named["messages"]) or [], + optional_params={ + key: value + for key, value in plan.named.items() + if value is not None and key not in ("model", "messages", "stream", "deployment_id") + }, + kwargs=dict(plan.kwargs), + logging_obj=plan.kwargs.get("litellm_logging_obj"), + custom_llm_provider=plan.custom_llm_provider, + stream=plan.named.get("stream") is True, + ) + return response if looped is None else looped + + +async def _aexecute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper: + loop: Final = asyncio.get_event_loop() + response: Final = await _settled_chat_response(plan, loop) + if isinstance(response, CustomStreamWrapper): + response.set_logging_event_loop(loop=loop) + return response + return await _with_agentic_loop(plan, response) + + +async def aexecute_planned_chat(plan: PlannedChat) -> ModelResponse | CustomStreamWrapper: + """Maps a provider failure raised while awaiting the call, as ``acompletion`` does.""" + try: + response: Final = await _aexecute_planned_chat(plan) + except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception acompletion raises + raise exception_type( + model=plan.model, + custom_llm_provider=plan.custom_llm_provider or "openai", + original_exception=e, + completion_kwargs=dict(plan.named), + extra_kwargs=dict(plan.kwargs), + ) from e + return _post_call_checked(plan, response, is_async=True) diff --git a/litellm/native/chat_providers/__init__.py b/litellm/native/chat_providers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/native/chat_providers/agents.py b/litellm/native/chat_providers/agents.py new file mode 100644 index 00000000000..5523ba0e6dc --- /dev/null +++ b/litellm/native/chat_providers/agents.py @@ -0,0 +1,273 @@ +from collections.abc import Mapping, Sequence +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +from pydantic import TypeAdapter + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.native.chat_providers.handlers import base_llm_http_handler +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.types.llms.openai import AllMessageValues + + +def complete_ragflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_a2a(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + ( + api_base, + api_key, + headers, + ) = litellm.A2AConfig.resolve_agent_config_from_registry( + agent_name=model, + api_base=api_base, + api_key=api_key, + headers=headers, + optional_params=optional_params, + ) + + # Fall back to environment variables and defaults + api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") + + if api_base is None: + raise Exception( + "api_base is required for A2A provider. " + "Either provide api_base parameter, set A2A_API_BASE environment variable, " + "or register the agent in the proxy with model='a2a/'." + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + + +def complete_langgraph(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + from litellm.llms.langgraph.chat.transformation import LangGraphConfig + + ( + api_base, + api_key, + ) = LangGraphConfig().get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + +def complete_microsoft_365_copilot(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + from litellm.llms.microsoft_365_copilot.chat.handler import acompletion, completion + from litellm.llms.microsoft_365_copilot.common_utils import as_secret_fields + + messages: Final[Sequence[AllMessageValues]] = cast( # cast-ok: dispatch messages are prevalidated + Sequence[AllMessageValues], + ctx.messages, + ) + optional_params_value: Final[object] = cast( # cast-ok: dispatch params are validated below + object, + ctx.optional_params, + ) + optional_params: Final = TypeAdapter(Mapping[str, object]).validate_python(optional_params_value) + kwargs_value: Final[object] = cast( # cast-ok: dispatch kwargs are validated below + object, + ctx.kwargs, + ) + kwargs: Final = TypeAdapter(Mapping[str, object]).validate_python(kwargs_value) + secret_fields: Final = as_secret_fields(kwargs.get("secret_fields")) + client: Final = dispatch_client_http(ctx) + if ctx.acompletion: + async_client: Final = client if isinstance(client, AsyncHTTPHandler) else None + return acompletion( + model=ctx.model, + messages=messages, + api_key=ctx.api_key, + litellm_params=ctx.litellm_params, + optional_params=optional_params, + secret_fields=secret_fields, + stream=ctx.stream is True, + timeout=ctx.timeout, + logging_obj=ctx.logging, + client=async_client, + shared_session=ctx.shared_session, + ) + sync_client: Final = client if isinstance(client, HTTPHandler) else None + return completion( + model=ctx.model, + messages=messages, + api_key=ctx.api_key, + litellm_params=ctx.litellm_params, + optional_params=optional_params, + secret_fields=secret_fields, + stream=ctx.stream is True, + timeout=ctx.timeout, + logging_obj=ctx.logging, + client=sync_client, + ) + + +def complete_langflow(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + from litellm.llms.langflow.chat.transformation import LangFlowConfig + + ( + api_base, + api_key, + ) = LangFlowConfig().get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) diff --git a/litellm/native/chat_providers/anthropic.py b/litellm/native/chat_providers/anthropic.py new file mode 100644 index 00000000000..e9d8106aa59 --- /dev/null +++ b/litellm/native/chat_providers/anthropic.py @@ -0,0 +1,132 @@ +import os +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.native.chat_providers.handlers import anthropic_chat_completions, base_llm_http_handler +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose +from litellm.secret_managers.main import get_secret_bool +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_anthropic_text( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + api_base = cast( + str | None, + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/complete", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if api_base is not None and not disable_url_suffix: + api_base = ensure_api_route(api_base, "v1/complete") + elif disable_url_suffix: + verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix") + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="anthropic_text", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def complete_anthropic(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + api_key = api_key or litellm.anthropic_key or litellm.api_key or os.environ.get("ANTHROPIC_API_KEY") + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # call /messages + # default route for all anthropic models + api_base = cast( + str | None, + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/messages", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if api_base is not None and not disable_url_suffix: + api_base = ensure_api_route(api_base, "v1/messages") + elif disable_url_suffix: + verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix") + + response: Final = anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + return response diff --git a/litellm/native/chat_providers/aws.py b/litellm/native/chat_providers/aws.py new file mode 100644 index 00000000000..2c5ae360bef --- /dev/null +++ b/litellm/native/chat_providers/aws.py @@ -0,0 +1,400 @@ +from collections.abc import Mapping +from functools import partial +from typing import Final + +import httpx +from pydantic import TypeAdapter + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( + AmazonAnthropicClaudeConfig, +) +from litellm.llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback +from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo, + NativeOutputFallback, + bedrock_route_for_request, + bedrock_strict_tools_fallback, + deployment_model_info, + without_bedrock_native_output_opt_in, + without_bedrock_route_prefix, +) +from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + bedrock_converse_chat_completion, + openai_like_chat_completion, + sagemaker_llm, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, get_secret + +_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str]) +_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object]) + + +def complete_bedrock_mantle( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") + api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") + headers = headers or litellm.headers + config: Final = bedrock_mantle_chat_config(model).get_config() + for k, v in provider_config_items(config): + if k not in optional_params: + optional_params[k] = v + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + +def complete_amazon_nova(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + api_key = ctx.api_key + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + api_key = api_key or litellm.amazon_nova_api_key or get_secret_str("AMAZON_NOVA_API_KEY") or litellm.api_key + api_base = ( + api_base or litellm.api_base or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1" + ) + return openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + ) + + +def complete_sagemaker_chat( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_sagemaker(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + custom_prompt_dict: Final = ctx.custom_prompt_dict + hf_model_name: Final = ctx.hf_model_name + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + + return sagemaker_llm.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + custom_prompt_dict=custom_prompt_dict, + hf_model_name=hf_model_name, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + acompletion=acompletion, + ) + + +_AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region_name", +) + + +def _native_output_fallback( + ctx: CompletionDispatchContext, + first_attempt_params: Mapping[str, object], + first_attempt_litellm_params: Mapping[str, object], +) -> NativeOutputFallback: + litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) + remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) + return NativeOutputFallback( + optional_params={ + **{key: value for key, value in remapped_params.items() if key != "aws_bedrock_client"}, + **{ + key: first_attempt_params[key] + for key in _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS + if key in first_attempt_params + }, + }, + litellm_params=litellm_params, + ) + + +def _converse_native_output_fallback( + ctx: CompletionDispatchContext, + first_attempt_params: Mapping[str, object], + first_attempt_litellm_params: Mapping[str, object], +) -> NativeOutputFallback | None: + fallback: Final = _native_output_fallback(ctx, first_attempt_params, first_attempt_litellm_params) + return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback + + +def _native_output_attempt_timeout( + ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None +) -> float | str | httpx.Timeout | None: + if fallback is None: + return ctx.timeout + retry_timeout: Final = ctx.timeout_for_optional_params(fallback.optional_params) + ctx.logging.record_applied_timeout(retry_timeout) + return retry_timeout + + +def complete_bedrock(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + if "aws_bedrock_client" in optional_params: + verbose_logger.warning( + "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." + ) + # Extract credentials for legacy boto3 client and pass thru to httpx + aws_bedrock_client: Final = optional_params.pop("aws_bedrock_client") + creds: Final = aws_bedrock_client._get_credentials().get_frozen_credentials() + + if creds.access_key: + optional_params["aws_access_key_id"] = creds.access_key + if creds.secret_key: + optional_params["aws_secret_access_key"] = creds.secret_key + if creds.token: + optional_params["aws_session_token"] = creds.token + if "aws_region_name" not in optional_params or optional_params["aws_region_name"] is None: + optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name + + additional_drop_params: Final = ( + _ADDITIONAL_DROP_PARAMS_ADAPTER.validate_python(ctx.kwargs["additional_drop_params"]) + if ctx.kwargs.get("additional_drop_params") is not None + else None + ) + bedrock_route: Final = bedrock_route_for_request(model, ctx.request_params, additional_drop_params) + if bedrock_route == "claude_platform": + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders.BEDROCK, + ) + model = BedrockModelInfo.get_claude_platform_model(model) + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + elif bedrock_route == "converse": + model = without_bedrock_route_prefix(model) + converse_model: Final = model + + def converse_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + return bedrock_converse_chat_completion.completion( + model=converse_model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + optional_params=optional_params if fallback is None else dict(fallback.optional_params), + litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + extra_headers=headers, # Use merged headers instead of original extra_headers + timeout=_native_output_attempt_timeout(ctx, fallback), + acompletion=acompletion, + client=client, + api_base=api_base, + api_key=api_key, + ) + + converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + response = call_with_native_output_fallback( + converse_attempt, + ( + partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) + if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) + else None + ), + converse_model, + strict_tools_fallback=( + partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params)) + if drops_unsupported_params(litellm_params.get("drop_params")) + else None + ), + ) + elif bedrock_route == "converse_like": + model = model.replace("converse_like/", "") + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + else: + + def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult: + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params if fallback is None else dict(fallback.optional_params), + litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params), + custom_llm_provider="bedrock", + timeout=_native_output_attempt_timeout(ctx, fallback), + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + response = call_with_native_output_fallback( + invoke_attempt, + ( + partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) + if isinstance(provider_config, AmazonAnthropicClaudeConfig) + and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) + else None + ), + model, + ) + + return response diff --git a/litellm/native/chat_providers/azure.py b/litellm/native/chat_providers/azure.py new file mode 100644 index 00000000000..2a6febe9114 --- /dev/null +++ b/litellm/native/chat_providers/azure.py @@ -0,0 +1,399 @@ +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import openai + +import litellm +from litellm.litellm_core_utils.prompt_templates.factory import stringify_json_tool_call_content +from litellm.llms.azure.azure import check_dynamic_azure_params +from litellm.llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url +from litellm.native.chat_providers.handlers import ( + azure_anthropic_chat_completions, + azure_chat_completions, + azure_o1_chat_completions, + azure_text_completions, + base_llm_http_handler, +) +from litellm.native.chat_providers.support import ( + dispatch_client_azure, + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_azure(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + _azure_detection_model: Final = ctx._azure_detection_model + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client: Final = dispatch_client_azure(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + max_retries: Final = ctx.max_retries + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + dynamic_params = False + if client is not None and (isinstance(client, openai.AzureOpenAI) or isinstance(client, openai.AsyncAzureOpenAI)): + dynamic_params = check_dynamic_azure_params( + azure_client_params={"api_version": api_version}, + azure_client=client, + ) + + api_type: Final = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") or litellm.AZURE_DEFAULT_API_VERSION + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( + "AZURE_AD_TOKEN" + ) + + azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) + azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + if max_retries is not None: + optional_params["max_retries"] = max_retries + + if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIO1Config.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + response = azure_o1_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + custom_llm_provider=custom_llm_provider, + ) + else: + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_azure_text(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client: Final = dispatch_client_azure(ctx) + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + api_type: Final = get_secret_str("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + + if api_base is None: + raise ValueError( + "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." + ) + + api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token: Final = optional_params.get("extra_body", {}).pop("azure_ad_token", None) or get_secret_str( + "AZURE_AD_TOKEN" + ) + + azure_ad_token_provider_value: Final = litellm_params.get("azure_ad_token_provider", None) + azure_ad_token_provider: Final = azure_ad_token_provider_value if callable(azure_ad_token_provider_value) else None + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## LOAD CONFIG - if set + config: Final = litellm.AzureOpenAIConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response: Final = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=cast(str, api_version), + api_type=api_type, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response + + +def complete_azure_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + + azure_ai_route: Final = AzureFoundryModelInfo.get_azure_ai_route(model) + + # Check if this is an agents route - model format: azure_ai/agents/ + if azure_ai_route == "agents": + from litellm.llms.azure_ai.agents import AzureAIAgentsConfig + + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + response = AzureAIAgentsConfig.completion( + model=model, + messages=messages, + api_base=api_base, + api_key=api_key, + model_response=model_response, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + acompletion=acompletion, + stream=stream, + headers=headers or litellm.headers, + ) + + # Check if this is a Claude model - route to Azure Anthropic handler + elif "claude" in model.lower(): + # Use Azure Anthropic handler for Claude models + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + if api_base: + api_base = azure_anthropic_messages_url(api_base) + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + else: + # Non-Claude models use standard Azure AI flow + api_base = AzureFoundryModelInfo.get_api_base(api_base) + # set API KEY + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## FOR COHERE + if "command-r" in model: # make sure tool call in messages are str + messages = stringify_json_tool_call_content(messages=messages) + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, # pass AsyncOpenAI, OpenAI client + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response diff --git a/litellm/native/chat_providers/custom.py b/litellm/native/chat_providers/custom.py new file mode 100644 index 00000000000..f4bdd09d9c6 --- /dev/null +++ b/litellm/native/chat_providers/custom.py @@ -0,0 +1,147 @@ +import time +from typing import Final + +import httpx + +import litellm +from litellm.exceptions import LiteLLMUnknownProvider +from litellm.llms.custom_llm import CustomLLM, custom_chat_llm_router +from litellm.native.chat_providers.support import get_encoding, print_verbose +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper + + +def _custom_api_first_output(resp: httpx.Response | None) -> str: + return resp.json()["data"][0]["output"][0] + + +def complete_custom(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = ctx.api_base + headers: Final = ctx.headers + kwargs: Final = ctx.kwargs + max_tokens: Final = ctx.max_tokens + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + temperature: Final = ctx.temperature + top_p: Final = ctx.top_p + + url: Final = litellm.api_base or api_base or "" + if url is None or url == "": + raise ValueError("api_base not set. Set api_base or litellm.api_base for custom endpoints") + + """ + assume input to custom LLM api bases follow this format: + resp = litellm.module_level_client.post( + api_base, + json={ + 'model': 'meta-llama/Llama-2-13b-hf', # model name + 'params': { + 'prompt': ["The capital of France is P"], + 'max_tokens': 32, + 'temperature': 0.7, + 'top_p': 1.0, + 'top_k': 40, + } + } + ) + + """ + prompt: Final = " ".join([message["content"] for message in messages]) + resp: Final = litellm.module_level_client.post( + url, + headers=headers, + json={ + "model": model, + "params": { + "prompt": [prompt], + "max_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "top_k": kwargs.get("top_k"), + }, + **kwargs.get("extra_body", {}), + }, + ) + """ + assume all responses from custom api_bases of this format: + { + 'data': [ + { + 'prompt': 'The capital of France is P', + 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], + 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], + 'message': 'ok' + } + ] + } + """ + string_response: Final = _custom_api_first_output(resp) + ## RESPONSE OBJECT + model_response.choices[0].message.content = string_response + model_response.created = int(time.time()) + model_response.model = model + return model_response + + +def complete_custom_providers( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = ctx.client + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + custom_handler: CustomLLM | None = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) + + ## ROUTE LLM CALL ## + handler_fn: Final = custom_chat_llm_router(async_fn=acompletion, stream=stream, custom_llm=custom_handler) + + headers = headers or litellm.headers or {} + + ## CALL FUNCTION + response: Final = handler_fn( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=get_encoding(), + ) + if stream is True: + return CustomStreamWrapper( + completion_stream=response, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract diff --git a/litellm/native/chat_providers/gateways.py b/litellm/native/chat_providers/gateways.py new file mode 100644 index 00000000000..48ce889a105 --- /dev/null +++ b/litellm/native/chat_providers/gateways.py @@ -0,0 +1,299 @@ +from typing import Final + +import openai + +import litellm +from litellm.constants import NADIR_DEFAULT_API_BASE +from litellm.native.chat_providers.handlers import base_llm_http_handler +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, provider_config_items +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_cometapi(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.cometapi_key or get_secret_str("COMETAPI_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret_str("COMETAPI_API_BASE") or "https://api.cometapi.com/v1" + + ## COMPLETION CALL + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + ## LOGGING + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def complete_openrouter(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret_str("OPENROUTER_API_KEY") + or get_secret_str("OR_API_KEY") + ) + + openrouter_site_url: Final = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name: Final = get_secret("OR_APP_NAME") or "liteLLM" + + openrouter_headers: Final = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + + _headers: Final = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) + + headers = openrouter_headers + + ## Load Config + config: Final = litellm.OpenrouterConfig.get_config() + for k, v in provider_config_items(config): + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response: Final = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="openrouter", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call(input=messages, api_key=openai.api_key, original_response=response) + + return response + + +def complete_nadir(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE + api_key: Final = ctx.api_key + + response: Final = base_llm_http_handler.completion( + model=ctx.model, + stream=ctx.stream, + messages=ctx.messages, + acompletion=ctx.acompletion, + api_base=api_base, + model_response=ctx.model_response, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + custom_llm_provider="nadir", + timeout=ctx.timeout, + headers=ctx.headers or litellm.headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=ctx.logging, + client=dispatch_client_http(ctx), + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + + return response + + +def complete_vercel_ai_gateway( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") + or "https://ai-gateway.vercel.sh/v1" + ) + + api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") + + vercel_site_url: Final = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" + vercel_app_name: Final = get_secret("VERCEL_APP_NAME") or "liteLLM" + + vercel_headers: Final = { + "http-referer": vercel_site_url, + "x-title": vercel_app_name, + } + + _headers: Final = headers or litellm.headers + if _headers: + vercel_headers.update(_headers) + + headers = vercel_headers + + ## Load Config + config: Final = litellm.VercelAIGatewayConfig.get_config() + for k, v in provider_config_items(config): + if k == "extra_body": + # we use openai 'extra_body' to pass vercel specific params - providerOptions + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response: Final = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="vercel_ai_gateway", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call(input=messages, api_key=openai.api_key, original_response=response) + + return response + + +def complete_edenai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = litellm.EdenAIChatConfig.get_api_base(ctx.api_base) + api_key: Final = litellm.EdenAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) + response: Final = base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + api_base=api_base, + custom_llm_provider="edenai", + model_response=ctx.model_response, + encoding=get_encoding(), + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + timeout=ctx.timeout, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + acompletion=ctx.acompletion, + stream=ctx.stream, + api_key=api_key, + headers=ctx.headers or litellm.headers, + client=dispatch_client_http(ctx), + provider_config=ctx.provider_config, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + return response + + +def complete_fal_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + if ctx.stream: + raise litellm.FalAIError( + status_code=400, + message="fal_ai chat completions do not support streaming", + ) + api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base) + api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) + response: Final = base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + api_base=api_base, + custom_llm_provider="fal_ai", + model_response=ctx.model_response, + encoding=get_encoding(), + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + timeout=ctx.timeout, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + acompletion=ctx.acompletion, + stream=ctx.stream, + api_key=api_key, + headers=ctx.headers or litellm.headers, + client=dispatch_client_http(ctx), + provider_config=ctx.provider_config, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + return response diff --git a/litellm/native/chat_providers/google.py b/litellm/native/chat_providers/google.py new file mode 100644 index 00000000000..edb1924be08 --- /dev/null +++ b/litellm/native/chat_providers/google.py @@ -0,0 +1,311 @@ +from typing import Final + +import litellm +from litellm.litellm_core_utils.core_helpers import safe_deep_copy +from litellm.llms.gemini.common_utils import get_api_key_from_env +from litellm.llms.vertex_ai import vertex_ai_non_gemini +from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + gdc_transformation, + vertex_chat_completion, + vertex_gemma_chat_completion, + vertex_model_garden_chat_completion, + vertex_partner_models_chat_completion, +) +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper, get_secret + + +def complete_vertex_ai_beta( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + vertex_ai_project: Final = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location: Final = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials: Final = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + gemini_api_key: Final = ( + api_key + or get_api_key_from_env() + or get_secret("PALM_API_KEY") # older palm api key should also work + or litellm.api_key + ) + + api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") + new_params: Final = safe_deep_copy(optional_params or {}) + return vertex_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=gemini_api_key, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + client=client, + api_base=api_base, + extra_headers=headers, + ) + + +def complete_vertex_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + vertex_ai_project: Final = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location: Final = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials: Final = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") + + new_params: Final = safe_deep_copy(optional_params or {}) + model_route: Final = get_vertex_ai_model_route(model=model, litellm_params=litellm_params) + + if model_route == VertexAIModelRoute.PARTNER_MODELS: + model_response = vertex_partner_models_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.GEMINI: + model_response = vertex_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=None, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + client=client, + api_base=api_base, + extra_headers=headers, + ) + elif model_route == VertexAIModelRoute.GEMMA: + # Vertex Gemma Models with custom prediction endpoint + model_response = vertex_gemma_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.MODEL_GARDEN: + # Vertex Model Garden - OpenAI compatible models + model_response = vertex_model_garden_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.AGENT_ENGINE: + # Vertex AI Agent Engine (Reasoning Engines) + from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + ) + + vertex_agent_engine_config: Final = VertexAgentEngineConfig() + + # Update litellm_params with vertex credentials + litellm_params["vertex_project"] = vertex_ai_project + litellm_params["vertex_location"] = vertex_ai_location + litellm_params["vertex_credentials"] = vertex_credentials + + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + model_response=model_response, + optional_params=new_params, + litellm_params=litellm_params, + encoding=get_encoding(), + api_key=None, + api_base=api_base, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, + custom_llm_provider="vertex_ai", + provider_config=vertex_agent_engine_config, + headers=headers or {}, + ) + else: # VertexAIModelRoute.NON_GEMINI + model_response = vertex_ai_non_gemini.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + + if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vertex_ai", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_gdc(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.gdc_key or get_secret_str("GDC_API_KEY") or litellm.api_key + api_base = api_base or litellm.gdc_api_base or get_secret_str("GDC_API_BASE") or litellm.api_base + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=gdc_transformation, + ) diff --git a/litellm/native/chat_providers/handlers.py b/litellm/native/chat_providers/handlers.py new file mode 100644 index 00000000000..773def0c8e0 --- /dev/null +++ b/litellm/native/chat_providers/handlers.py @@ -0,0 +1,51 @@ +from typing import Final + +from litellm.llms.anthropic.chat import AnthropicChatCompletion +from litellm.llms.azure.azure import AzureChatCompletion +from litellm.llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion +from litellm.llms.azure.completion.handler import AzureTextCompletion +from litellm.llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion +from litellm.llms.bedrock.chat import BedrockConverseLLM +from litellm.llms.bytez.chat.transformation import BytezChatConfig +from litellm.llms.codestral.completion.handler import CodestralTextCompletion +from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.gdc.chat.transformation import GDCGeminiConfig +from litellm.llms.lemonade.chat.transformation import LemonadeChatConfig +from litellm.llms.openai.completion.handler import OpenAITextCompletion +from litellm.llms.openai.openai import OpenAIChatCompletion +from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler +from litellm.llms.ovhcloud.chat.transformation import OVHCloudChatConfig +from litellm.llms.predibase.chat.handler import PredibaseChatCompletion +from litellm.llms.sagemaker.completion.handler import SagemakerLLM +from litellm.llms.sap.chat.handler import GenAIHubOrchestration +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels +from litellm.llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels +from litellm.llms.vertex_ai.vertex_model_garden.main import VertexAIModelGardenModels +from litellm.llms.watsonx.chat.handler import WatsonXChatHandler + +openai_chat_completions = OpenAIChatCompletion() +openai_text_completions: Final = OpenAITextCompletion() +sap_gen_ai_hub_chat_completions: Final = GenAIHubOrchestration() +anthropic_chat_completions: Final = AnthropicChatCompletion() +azure_anthropic_chat_completions: Final = AzureAnthropicChatCompletion() +azure_chat_completions = AzureChatCompletion() +azure_o1_chat_completions: Final = AzureOpenAIO1ChatCompletion() +azure_text_completions: Final = AzureTextCompletion() +predibase_chat_completions: Final = PredibaseChatCompletion() +codestral_text_completions: Final = CodestralTextCompletion() +bedrock_converse_chat_completion: Final = BedrockConverseLLM() +vertex_chat_completion: Final = VertexLLM() +vertex_partner_models_chat_completion: Final = VertexAIPartnerModels() +vertex_gemma_chat_completion: Final = VertexAIGemmaModels() +vertex_model_garden_chat_completion: Final = VertexAIModelGardenModels() +gdc_transformation: Final = GDCGeminiConfig() +sagemaker_llm: Final = SagemakerLLM() +watsonx_chat_completion: Final = WatsonXChatHandler() +openai_like_chat_completion: Final = OpenAILikeChatHandler() +base_llm_http_handler = BaseLLMHTTPHandler() +base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() +bytez_transformation: Final = BytezChatConfig() +ovhcloud_transformation: Final = OVHCloudChatConfig() +lemonade_transformation: Final = LemonadeChatConfig() diff --git a/litellm/native/chat_providers/model_labs.py b/litellm/native/chat_providers/model_labs.py new file mode 100644 index 00000000000..773bb6e4eaa --- /dev/null +++ b/litellm/native/chat_providers/model_labs.py @@ -0,0 +1,926 @@ +import os +from types import MappingProxyType +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.cohere.common_utils import CohereModelInfo +from litellm.llms.deprecated_providers import aleph_alpha +from litellm.llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion +from litellm.llms.replicate.chat.handler import completion as replicate_chat_completion +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + bytez_transformation, + codestral_text_completions, + openai_like_chat_completion, + predibase_chat_completions, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper, get_secret + + +def complete_deepseek(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_fireworks_ai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + optional_params: Final = ( + provider_config.map_extra_body_params(optional_params=ctx.optional_params, model=model) + if isinstance(provider_config, litellm.FireworksAIConfig) + else ctx.optional_params + ) + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_together_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args=MappingProxyType({"headers": headers}), + ) + raise + + return response + + +def complete_scaledown(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + try: + return base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + headers=ctx.headers, + model_response=ctx.model_response, + api_key=ctx.api_key, + api_base=ctx.api_base, + acompletion=ctx.acompletion, + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value + float | httpx.Timeout, ctx.timeout + ), + client=dispatch_client_http(ctx), + custom_llm_provider=ctx.custom_llm_provider, + encoding=get_encoding(), + stream=ctx.stream, + provider_config=ctx.provider_config, + ) + except Exception as exc: + ctx.logging.post_call( + input=ctx.messages, + api_key=ctx.api_key, + original_response=str(exc), + additional_args={"headers": ctx.headers}, + ) + raise + + +def complete_xai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_groq(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("GROQ_API_BASE") + or "https://api.groq.com/openai/v1" + ) + + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.groq_key + or get_secret("GROQ_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config: Final = litellm.GroqChatConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_opencode_go(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + return base_llm_http_handler.completion( # pyright: ignore[reportUnknownMemberType] # shared legacy handler has untyped request mappings + model=ctx.model, + stream=ctx.stream, + messages=ctx.messages, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # existing dispatch context carries validated messages + acompletion=ctx.acompletion, + api_base=ctx.api_base, + model_response=ctx.model_response, + optional_params=ctx.optional_params, # pyright: ignore[reportUnknownMemberType] # existing dispatch context carries the mapped provider parameters + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + custom_llm_provider=ctx.custom_llm_provider, + timeout=ctx.timeout, # pyright: ignore[reportArgumentType] # completion normalizes timeout before constructing this context + headers=ctx.headers, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # legacy context retains caller headers + encoding=get_encoding(), + api_key=ctx.api_key, + logging_obj=ctx.logging, + client=dispatch_client_http(ctx), + ) + + +def complete_minimax(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/v1" + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def complete_mistral(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") + api_base = api_base or litellm.api_base or get_secret("MISTRAL_API_BASE") or "https://api.mistral.ai/v1" + + return base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + + +def complete_replicate(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + replicate_key: Final = ( + api_key + or litellm.replicate_key + or litellm.api_key + or get_secret("REPLICATE_API_KEY") + or get_secret("REPLICATE_API_TOKEN") + ) + + api_base = api_base or litellm.api_base or get_secret("REPLICATE_API_BASE") or "https://api.replicate.com/v1" + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = replicate_chat_completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), # for calculating input/output tokens + api_key=replicate_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + acompletion=acompletion, + headers=headers, + ) + + if optional_params.get("stream", False) is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=replicate_key, + original_response=model_response, + ) + + return model_response + + +def complete_nlp_cloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + + nlp_cloud_key: Final = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret("NLP_CLOUD_API_BASE") or "https://api.nlpcloud.io/v1/gpu/" + + response = nlp_cloud_chat_completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=nlp_cloud_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="nlp_cloud", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_aleph_alpha(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + api_key: Final = ctx.api_key + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + aleph_alpha_key: Final = ( + api_key + or litellm.aleph_alpha_key + or get_secret("ALEPH_ALPHA_API_KEY") + or get_secret("ALEPHALPHA_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base or litellm.api_base or get_secret("ALEPH_ALPHA_API_BASE") or "https://api.aleph-alpha.com/complete" + ) + + model_response = aleph_alpha.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + default_max_tokens_to_sample=litellm.max_tokens, + api_key=aleph_alpha_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="aleph_alpha", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_cohere_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + cohere_key: Final = ( + api_key + or litellm.cohere_key + or get_secret_str("COHERE_API_KEY") + or get_secret_str("CO_API_KEY") + or litellm.api_key + ) + + cohere_route: Final = CohereModelInfo.get_cohere_route(model) + verbose_logger.debug("Cohere route: %s", cohere_route) + # Set API base based on route + if cohere_route == "v2": + api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.com/v2/chat" + # Remove v2/ prefix from model name for the actual API call + if "v2/" in model: + model = model.replace("v2/", "") + else: + api_base = api_base or litellm.api_base or get_secret_str("COHERE_API_BASE") or "https://api.cohere.ai/v1/chat" + + headers = headers or litellm.headers or {} + if headers is None: + headers = {} + + if extra_headers is not None: + headers.update(extra_headers) + + verbose_logger.debug("Model: %s, API Base: %s", model, api_base) + verbose_logger.debug("Provider Config: %s", provider_config) + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cohere_chat", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=cohere_key, + provider_config=provider_config, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def complete_maritalk(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + api_key: Final = ctx.api_key + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + + maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key + + api_base = api_base or litellm.api_base or get_secret("MARITALK_API_BASE") or "https://chat.maritaca.ai/api" + + return openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + api_key=maritalk_key, + logging_obj=logging, + custom_llm_provider="maritalk", + custom_prompt_dict=custom_prompt_dict, + ) + + +def complete_huggingface(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + huggingface_key: Final = ( + api_key + or litellm.huggingface_key + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_API_KEY") + or litellm.api_key + ) + hf_headers: Final = headers or litellm.headers + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=hf_headers, + model_response=model_response, + api_key=huggingface_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + +def complete_compactifai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key + + api_base = api_base or "https://api.compactif.ai/v1" + + ## COMPLETION CALL + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + +def complete_predibase(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + tenant_id: Final = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + if tenant_id is None: + raise ValueError( + "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." + ) + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = api_key or litellm.api_key or litellm.predibase_key or get_secret("PREDIBASE_API_KEY") + + _model_response: Final = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + timeout=timeout, + ) + + if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: + return _model_response + return _model_response + + +def complete_text_completion_codestral( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict: Final = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or "https://codestral.mistral.ai/v1/fim/completions" + ) + + api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") + + text_completion_model_response: Final = litellm.TextCompletionResponse(stream=stream) + + _model_response: Final = codestral_text_completions.completion( + model=model, + messages=messages, + model_response=text_completion_model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + timeout=timeout, + ) + + if "stream" in optional_params and optional_params["stream"] is True and acompletion is False: + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_bytez(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.bytez_key or get_secret_str("BYTEZ_API_KEY") or litellm.api_key + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=bytez_transformation, + ) + + return response diff --git a/litellm/native/chat_providers/openai.py b/litellm/native/chat_providers/openai.py new file mode 100644 index 00000000000..7a569638d98 --- /dev/null +++ b/litellm/native/chat_providers/openai.py @@ -0,0 +1,390 @@ +from typing import Final, cast # noqa: TID251 # casts moved verbatim from litellm/main.py + +import openai + +import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import ensure_api_route +from litellm.llms.base_llm.chat.transformation import with_attribution_headers +from litellm.llms.custom_httpx.http_handler import http2_enabled +from litellm.native.chat_providers.handlers import ( + base_llm_aiohttp_handler, + base_llm_http_handler, + openai_chat_completions, + openai_text_completions, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + dispatch_client_openai, + dispatch_metadata, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.responses.bridge_check import resolve_openai_api_base +from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_requester_metadata, get_secret + + +def complete_text_completion_openai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_openai(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + text_completion: Final = ctx.text_completion + timeout: Final = ctx.timeout + + openai.api_type = "openai" + + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + openai.api_version = None + # set API KEY + + api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config: Final = litellm.OpenAITextCompletionConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: + openai.organization = litellm.organization + + ## COMPLETION CALL + _response = openai_text_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + acompletion=acompletion, + client=client, # pass AsyncOpenAI, OpenAI client + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + ) + + if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: + # convert to chat completion response + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_aiohttp_openai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + if http2_enabled(): + verbose_logger.warning( + "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1" + ) + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + extra_headers: Final = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + return base_llm_aiohttp_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + +def complete_custom_openai( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + custom_prompt_dict: Final = ctx.custom_prompt_dict + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + metadata: Final = dispatch_metadata(ctx) + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + organization = ctx.organization + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + api_base = resolve_openai_api_base(api_base) + if custom_llm_provider == "bigmodel" and not api_key: + raise litellm.AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + organization = ( + organization + or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION")) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + openai.organization = organization + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + outbound_headers: Final = ( + headers + if provider_config is None + else with_attribution_headers(provider_config.get_attribution_headers(), headers) + ) + + # Add GitHub Copilot headers (same as /responses endpoint does) + if custom_llm_provider == "github_copilot": + from litellm.llms.github_copilot.authenticator import Authenticator + from litellm.llms.github_copilot.common_utils import ( + get_copilot_default_headers, + pin_session_authorization, + ) + from litellm.llms.github_copilot.per_user_auth import ( + require_github_copilot_user_session, + ) + + user_session: Final = require_github_copilot_user_session(litellm_params) + copilot_headers: Final = get_copilot_default_headers( + user_session.token if user_session is not None else Authenticator().get_api_key() + ) + if extra_headers: + copilot_headers.update( + cast("dict[str, str]", extra_headers) # cast-ok: extra_headers is a str-valued request dict + ) + if user_session is not None: + pin_session_authorization(copilot_headers, user_session.token) + extra_headers = copilot_headers + + use_base_llm_http_handler: Final = get_secret_bool("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER") + + if extra_headers is not None and not use_base_llm_http_handler: + optional_params["extra_headers"] = extra_headers + + if litellm.enable_preview_features and metadata is not None: # [PREVIEW] allow metadata to be passed to OPENAI + openai_metadata: Final = get_requester_metadata(metadata) + if openai_metadata is not None: + optional_params["metadata"] = openai_metadata + + ## LOAD CONFIG - if set + config: Final = litellm.OpenAIConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + try: + if use_base_llm_http_handler: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=outbound_headers, + client=client, + provider_config=provider_config, + ) + else: + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=outbound_headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, + custom_llm_provider=custom_llm_provider, + shared_session=shared_session, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": outbound_headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": outbound_headers}, + ) + + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_text_completion_inception( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_openai(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + text_completion: Final = ctx.text_completion + timeout: Final = ctx.timeout + + passed_api_base: Final = api_base or optional_params.pop("api_base", None) or optional_params.pop("base_url", None) + api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = ensure_api_route(api_base, "fim") + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + ) + + if optional_params.get("stream", False) is False and acompletion is False and text_completion is False: + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract diff --git a/litellm/native/chat_providers/platforms.py b/litellm/native/chat_providers/platforms.py new file mode 100644 index 00000000000..7d793d5ab3b --- /dev/null +++ b/litellm/native/chat_providers/platforms.py @@ -0,0 +1,623 @@ +from typing import Final + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.databricks.common_utils import DatabricksBase +from litellm.native.chat_providers.handlers import ( + base_llm_http_handler, + ovhcloud_transformation, + sap_gen_ai_hub_chat_completions, + watsonx_chat_completion, +) +from litellm.native.chat_providers.support import ( + dispatch_client_http, + get_encoding, + print_verbose, + provider_config_items, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import get_secret + + +def complete_heroku(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_gigachat(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret("GIGACHAT_API_KEY") + or get_secret("GIGACHAT_CREDENTIALS") + ) + + headers = headers or litellm.headers or {} + + ## COMPLETION CALL + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_sap(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + headers = headers or litellm.headers + ## LOAD CONFIG - if set + config: Final = litellm.GenAIHubOrchestrationConfig.get_config() + for k, v in provider_config_items(config): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + return sap_gen_ai_hub_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + shared_session=shared_session, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + api_key=api_key, + api_base=api_base, + stream=stream, + ) + + +def complete_oci(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + +def complete_databricks(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = DatabricksBase.resolve_api_base(ctx.api_base) + api_key: Final = DatabricksBase.resolve_api_key(ctx.api_key) + client: Final = dispatch_client_http(ctx) + headers = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + headers = headers or litellm.headers + + ## COMPLETION CALL + try: + response: Final = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="databricks", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response + + +def complete_databricks_agent(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + from litellm.llms.databricks.agent.transformation import DatabricksAgentConfig + + api_base, api_key = DatabricksAgentConfig.resolve_api_base_and_key( + api_base=ctx.api_base or litellm.api_base, + api_key=ctx.api_key or litellm.api_key, + ) + return base_llm_http_handler.completion( + model=ctx.model, + stream=ctx.stream, + messages=ctx.messages, + acompletion=ctx.acompletion, + api_base=api_base, + model_response=ctx.model_response, + optional_params=ctx.optional_params, + litellm_params={**ctx.litellm_params, "api_base": api_base}, + shared_session=ctx.shared_session, + custom_llm_provider=ctx.custom_llm_provider, + timeout=ctx.timeout, + headers=ctx.headers or litellm.headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=ctx.logging, + client=dispatch_client_http(ctx), + ) + + +def complete_datarobot(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + return base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + +def complete_watsonx(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_prompt_dict: Final = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout + + return watsonx_chat_completion.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=get_encoding(), + custom_llm_provider="watsonx", + ) + + +def complete_watsonx_text( + ctx: CompletionDispatchContext, +) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = ( + api_key + or optional_params.pop("apikey", None) + or get_secret_str("WATSONX_APIKEY") + or get_secret_str("WATSONX_API_KEY") + or get_secret_str("WX_API_KEY") + ) + + api_base = ( + api_base + or optional_params.pop( + "url", + optional_params.pop("api_base", optional_params.pop("base_url", None)), + ) + or get_secret_str("WATSONX_API_BASE") + or get_secret_str("WATSONX_URL") + or get_secret_str("WX_URL") + or get_secret_str("WML_URL") + ) + + wx_credentials: Final = optional_params.pop( + "wx_credentials", + optional_params.pop("watsonx_credentials", None), # follow {provider}_credentials, same as vertex ai + ) + + token: str | None = None + if wx_credentials is not None: + api_base = wx_credentials.get("url", api_base) + api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key)) + token = wx_credentials.get( + "token", + wx_credentials.get( + "watsonx_token", None + ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' + ) + + if token is not None: + optional_params["token"] = token + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="watsonx_text", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_cloudflare(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.cloudflare_api_key or litellm.api_key or get_secret("CLOUDFLARE_API_KEY") + api_base = api_base or litellm.api_base or get_secret("CLOUDFLARE_API_BASE") + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cloudflare", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + +def complete_snowflake(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key: Final = ctx.api_key + injected_client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + client: Final = ( + injected_client if injected_client is not None else (HTTPHandler(timeout=timeout) if stream is False else None) + ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible + + try: + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + ) + + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def complete_gradient_ai(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="gradient_ai", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + ) + + +def complete_ovhcloud(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.ovhcloud_key or get_secret_str("OVHCLOUD_API_KEY") or litellm.api_key + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OVHCLOUD_API_BASE") + or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" + ) + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=ovhcloud_transformation, + ) + + return response diff --git a/litellm/native/chat_providers/self_hosted.py b/litellm/native/chat_providers/self_hosted.py new file mode 100644 index 00000000000..0f8d6338a51 --- /dev/null +++ b/litellm/native/chat_providers/self_hosted.py @@ -0,0 +1,329 @@ +import os +from typing import Final + +from pydantic import TypeAdapter + +import litellm +from litellm.llms.oobabooga.chat import oobabooga +from litellm.llms.petals.completion import handler as petals_handler +from litellm.llms.vllm.completion import handler as vllm_handler +from litellm.native.chat_providers.handlers import base_llm_http_handler, lemonade_transformation +from litellm.native.chat_providers.support import dispatch_client_http, get_encoding, print_verbose +from litellm.secret_managers.main import get_secret_str +from litellm.types.completion import CompletionDispatchContext, CompletionDispatchResult +from litellm.utils import CustomStreamWrapper, get_secret + +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) + + +def complete_hosted_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + provider_config: Final = ctx.provider_config + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def complete_oobabooga(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base: Final = ctx.api_base + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + model_response = oobabooga.completion( + model=model, + messages=messages, + model_response=model_response, + api_base=api_base, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="oobabooga", + logging_obj=logging, + ) + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + + +def complete_vllm(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = vllm_handler.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] + # don't try to access stream object, + return CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vllm", + logging_obj=logging, + ) + + ## RESPONSE OBJECT + return model_response + + +def complete_ollama(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_ollama_chat(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434" + + api_key = api_key or litellm.ollama_key or os.environ.get("OLLAMA_API_KEY") or litellm.api_key + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" + + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama_chat", + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + +def complete_triton(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base = ctx.api_base + api_key: Final = ctx.api_key + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + shared_session: Final = ctx.shared_session + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_base = litellm.api_base or api_base + return base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=get_encoding(), + api_key=api_key, + logging_obj=logging, + ) + + +def complete_petals(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + api_base = ctx.api_base + client: Final = dispatch_client_http(ctx) + litellm_params: Final = ctx.litellm_params + logger_fn: Final = ctx.logger_fn + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response = ctx.model_response + optional_params: Final = ctx.optional_params + stream = ctx.stream + + api_base = api_base or litellm.api_base + + stream = optional_params.pop("stream", False) + model_response = petals_handler.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=get_encoding(), + logging_obj=logging, + client=client, + ) + if stream is True: ## [BETA] + # Fake streaming for petals + resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"]) + return CustomStreamWrapper( + resp_string, + model, + custom_llm_provider="petals", + logging_obj=logging, + ) + return model_response + + +def complete_lemonade(ctx: CompletionDispatchContext) -> CompletionDispatchResult: + acompletion: Final = ctx.acompletion + api_base: Final = ctx.api_base + api_key = ctx.api_key + client: Final = dispatch_client_http(ctx) + custom_llm_provider: Final = ctx.custom_llm_provider + headers: Final = ctx.headers + litellm_params: Final = ctx.litellm_params + logging: Final = ctx.logging + messages: Final = ctx.messages + model: Final = ctx.model + model_response: Final = ctx.model_response + optional_params: Final = ctx.optional_params + stream: Final = ctx.stream + timeout: Final = ctx.timeout + + api_key = api_key or litellm.lemonade_key or get_secret_str("LEMONADE_API_KEY") or litellm.api_key + + response: Final = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=get_encoding(), + stream=stream, + provider_config=lemonade_transformation, + ) + + return response diff --git a/litellm/native/chat_providers/support.py b/litellm/native/chat_providers/support.py new file mode 100644 index 00000000000..a36b81b0c32 --- /dev/null +++ b/litellm/native/chat_providers/support.py @@ -0,0 +1,47 @@ +from collections.abc import Iterable, Mapping +from functools import cache + +import openai + +import litellm +from litellm._lazy_imports import get_default_encoding +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.completion import CompletionDispatchContext + + +def dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: + return ctx.metadata + + +def dispatch_client_http(ctx: CompletionDispatchContext) -> HTTPHandler | AsyncHTTPHandler | None: + return ctx.client + + +def dispatch_client_azure( + ctx: CompletionDispatchContext, +) -> openai.AzureOpenAI | openai.AsyncAzureOpenAI | HTTPHandler | AsyncHTTPHandler | None: + return ctx.client + + +def dispatch_client_openai(ctx: CompletionDispatchContext) -> openai.OpenAI | openai.AsyncOpenAI | None: + return ctx.client + + +def provider_config_items(config: Mapping[str, object]) -> Iterable[tuple[str, object]]: + return config.items() + + +def print_verbose(print_statement: object) -> None: + try: + verbose_logger.debug(print_statement) + if litellm.set_verbose: + print(print_statement) # noqa: T201 + except Exception: + pass + + +@cache +def get_encoding() -> Tokenizer: + return get_default_encoding() diff --git a/litellm/native/messages.py b/litellm/native/messages.py new file mode 100644 index 00000000000..491db765c1b --- /dev/null +++ b/litellm/native/messages.py @@ -0,0 +1,120 @@ +"""Sends an Anthropic Messages request to a provider that speaks Messages natively.""" + +from collections.abc import AsyncIterator, Coroutine, Mapping +from typing import Final + +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.router_utils.reasoning_effort_capability import ( + RequestedReasoningEffort, + deployment_reasoning_effort_levels, + keeps_effort, + model_group_from_metadata, + resolve_deployment_reasoning_effort, + with_adaptive_thinking_as_effort, +) +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from litellm.types.router import GenericLiteLLMParams + +__all__ = ("execute_messages",) + + +def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]: + rest: Final = {key: value for key, value in params.items() if key != "output_config"} + return rest if output_config is None else {**rest, "output_config": output_config} + + +def _with_deployment_output_effort( + params: Mapping[str, object], + *, + model: str, + custom_llm_provider: str, + kwargs: Mapping[str, object], + drop_params: bool, +) -> Mapping[str, object]: + deployment_model_info: Final = kwargs.get("model_info") + + def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort: + return resolve_deployment_reasoning_effort( + requested, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")), + drop_params=drop_params, + ).value + + def translate(source: Mapping[str, object]) -> Mapping[str, object]: + return with_adaptive_thinking_as_effort( + source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) + + requested_output: Final = as_str_mapping(params.get("output_config")) + caller_output: Final = None if requested_output is None else resolve(requested_output) + caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output) + if deployment_reasoning_effort_levels(deployment_model_info) is None: + return caller_params + if requested_output is not None and keeps_effort(requested_output, caller_output): + return _with_output_config(translate(params), caller_output) + translated: Final = translate(caller_params) + budget_output: Final = as_str_mapping(translated.get("output_config")) + return translated if budget_output is None else _with_output_config(translated, resolve(budget_output)) + + +def execute_messages( + *, + http_handler: BaseLLMHTTPHandler, + model: str, + messages: list[dict[str, object]], # mutable-ok: the HTTP handler takes list[dict] messages + provider_config: BaseAnthropicMessagesConfig, + request_params: Mapping[str, object], + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + api_key: str | None, + api_base: str | None, + dynamic_api_base: str | None, + client: AsyncHTTPHandler | None, + is_async: bool, + stream: bool | None, + request_kwargs: dict[str, object], # mutable-ok: forwarded as the HTTP handler's kwargs dict +) -> AnthropicMessagesResponse | Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]]: + optional_params: Final = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params=request_params, + model=model, + drop_params=litellm_params.get("drop_params") is True, + custom_llm_provider=custom_llm_provider, + ) + resolved_api_base: Final = ( + dynamic_api_base + if dynamic_api_base is not None and provider_config.uses_get_llm_provider_api_base() + else api_base + ) + return http_handler.anthropic_messages_handler( + model=model, + messages=messages, + anthropic_messages_provider_config=provider_config, + anthropic_messages_optional_request_params=dict( + _with_deployment_output_effort( + optional_params, + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=request_kwargs, + drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True, + ) + ), + _is_async=is_async, + client=client, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=resolved_api_base, + stream=stream, + kwargs=request_kwargs, + ) diff --git a/litellm/native/request_support.py b/litellm/native/request_support.py new file mode 100644 index 00000000000..1473e9673fa --- /dev/null +++ b/litellm/native/request_support.py @@ -0,0 +1,149 @@ +from collections.abc import Callable, Mapping +from typing import Final, Protocol + +from typing_extensions import assert_never + +import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_litellm_params import InvalidControlOption, parse_control_options +from litellm.responses.bridge_check import keep_registered_mode +from litellm.types.litellm_params import ControlOptions +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.utils import CustomPricingLiteLLMParams + + +def resolve_control_options(kwargs: Mapping[str, object], model: str) -> ControlOptions: + control: Final = parse_control_options(kwargs) + match control: + case ControlOptions(): + return control + case InvalidControlOption(param=param, message=message): + if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True: + return ControlOptions() + raise litellm.BadRequestError(message=message, model=model, llm_provider=None, body={"param": param}) + case _: + return assert_never(control) + + +class _AdmissionReservation(LiteLLMBaseModel): + input_tokens: int | None = None + + +class _AdmissionMetadata(LiteLLMBaseModel): + user_api_key_budget_reservation: _AdmissionReservation | None = None + + +def admission_input_tokens(kwargs: Mapping[str, object]) -> int | None: + reservations: Final = ( + _AdmissionMetadata.model_validate(kwargs.get(key) or {}).user_api_key_budget_reservation + for key in ("litellm_metadata", "metadata") + ) + return next( + ( + reservation.input_tokens + for reservation in reservations + if reservation and reservation.input_tokens is not None + ), + None, + ) + + +def admitted_prompt_token_counter(prompt_tokens: int | None) -> Callable[[], int] | None: + if prompt_tokens is None: + return None + return lambda: prompt_tokens + + +class _ProxyAuthHeadersProvider(Protocol): + def get_auth_headers(self) -> Mapping[str, str]: ... + + +def proxy_auth_headers(proxy_auth: _ProxyAuthHeadersProvider) -> Mapping[str, str]: + return proxy_auth.get_auth_headers() + + +def _build_custom_pricing_entry( + custom_llm_provider: str, + kwargs: dict, + model_info: dict | None = None, +) -> dict: + """Build a complete model cost entry from kwargs and model_info. + + Collects all CustomPricingLiteLLMParams fields present in kwargs and + merges metadata from model_info (mode, supports_prompt_caching, max_tokens) + so that register_model() receives the full pricing configuration. + """ + entry: Final[dict] = {"litellm_provider": custom_llm_provider} + + for field_name in CustomPricingLiteLLMParams.model_fields: + value = kwargs.get(field_name) + if value is not None: + entry[field_name] = value + + if model_info and isinstance(model_info, dict): + for key in ("mode", "supports_prompt_caching", "max_tokens"): + if key in model_info and model_info[key] is not None: + entry.setdefault(key, model_info[key]) + + return entry + + +def _get_router_deployment_id(kwargs: dict) -> str | None: + for metadata_key in ("litellm_metadata", "metadata"): + metadata = kwargs.get(metadata_key) or {} + if not isinstance(metadata, dict): + continue + deployment_model_info = metadata.get("model_info") or {} + if not isinstance(deployment_model_info, dict): + continue + deployment_id = deployment_model_info.get("id") + if deployment_id is not None: + return str(deployment_id) + return None + + +def register_custom_pricing_for_request( + model: str, + custom_llm_provider: str, + kwargs: dict, + model_info: dict | None, +) -> None: + """Register per-request custom pricing in litellm.model_cost. + + Router-originated requests (identified by the deployment id the router puts + in metadata) get their full pricing registered under that unique id only; + the shared ``{provider}/{model}`` key receives the entry with pricing fields + stripped, mirroring Router._create_deployment. This keeps one deployment's + pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in + pricing used by sibling deployments of the same backend model. Direct SDK + calls keep the legacy behavior of registering the shared key with pricing. + Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on + deployment registration, since a call routes by its own ``model_info``. + """ + entry: Final = _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) + shared_key: Final = f"{custom_llm_provider}/{model}" + deployment_id: Final = _get_router_deployment_id(kwargs) + if deployment_id is None: + litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) + return + shared_entry: Final = keep_registered_mode( + shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) + ) + litellm.register_model( + {deployment_id: entry}, + persist_across_reloads=False, + warning_display_name=shared_key, + custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, + ) + litellm.register_model( + {shared_key: shared_entry}, + persist_across_reloads=False, + warning_display_name=shared_key, + custom_llm_provider=custom_llm_provider, + add_to_provider_model_sets=False, + ) diff --git a/litellm/native/responses.py b/litellm/native/responses.py new file mode 100644 index 00000000000..37f512f810e --- /dev/null +++ b/litellm/native/responses.py @@ -0,0 +1,185 @@ +"""Sends a Responses API request to a provider that speaks Responses natively.""" + +from collections.abc import Collection, Coroutine, Mapping, Sequence +from typing import TYPE_CHECKING, Annotated, Final + +import httpx +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter + +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind +from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.data_residency import infer_openai_data_residency +from litellm.responses.litellm_completion_transformation.compaction_items import ( + with_compaction_items_as_summaries, +) +from litellm.responses.litellm_completion_transformation.transformation import ( + with_compaction_items_as_params, +) +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import CostPerToken + +if TYPE_CHECKING: + from aiohttp import ClientSession + +__all__ = ("execute_responses",) + + +class _DeploymentModelInfo(BaseModel): + model_config = ConfigDict(frozen=True) + id: object = None + + +_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter( + Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")] +) + + +def _is_router_deployment(deployment_model_info: object) -> bool: + """A Router stamps the deployment's model_info (with its id) onto every call it routes.""" + parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info) + return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None + + +_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) + + +def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None: + if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None: + return None + return _COST_PER_TOKEN.validate_python( + litellm_params.model_dump( + include={ + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + }, + exclude_none=True, + ) + ) + + +def execute_responses( + *, + http_handler: BaseLLMHTTPHandler, + model: str, + input: str | ResponseInputParam, + provider_config: BaseResponsesAPIConfig, + requested_params: ResponsesAPIOptionalRequestParams, + allowed_openai_params: Sequence[str] | None, + declared_reasoning_efforts: Collection[str], + deployment_model_info: object, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + user: str | None, + extra_headers: dict[str, object] | None, + extra_body: Mapping[str, object] | None, + timeout: float | httpx.Timeout | None, + is_async: bool, + stream: bool | None, + client: HTTPHandler | AsyncHTTPHandler | None, + shared_session: "ClientSession | None", + litellm_metadata: dict[str, object] | None, + request_kwargs: dict[str, object], +) -> ( + ResponsesAPIResponse + | BaseResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] +): + request_params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=provider_config, + response_api_optional_params=requested_params, + allowed_openai_params=allowed_openai_params, + drop_params=normalize_drop_params(request_kwargs.get("drop_params")), + declared_reasoning_efforts=declared_reasoning_efforts, + ) + + logging_obj.update_from_kwargs( + kwargs=request_kwargs, + model=model, + user=user, + optional_params=dict(request_params), + litellm_params={ + **request_params, + **litellm_params.custom_pricing_overrides(), + "aresponses": is_async, + "litellm_call_id": request_kwargs.get("litellm_call_id"), + "model_info": request_kwargs.get("model_info"), + "data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base), + "metadata": ( + request_kwargs["litellm_metadata"] + if "litellm_metadata" in request_kwargs + else request_kwargs.get("metadata") + ), + "provider_affinity_header": litellm_params.provider_affinity_header, + }, + custom_llm_provider=custom_llm_provider, + ) + if not _is_router_deployment(deployment_model_info): + logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) # rebind-ok: prices the call + logging_obj.custom_cost_per_second = litellm_params.cost_per_second # rebind-ok: prices the call + + fake_stream: Final = provider_config.should_fake_stream( + model=model, stream=stream, custom_llm_provider=custom_llm_provider + ) + applied_timeout: Final = CompletionTimeout.for_stream( + (StreamKind.FAKED if fake_stream else StreamKind.LIVE) if stream is True else StreamKind.NOT_STREAMED, + CompletionTimeout.as_limit(timeout), + None if stream is not True or litellm_params.stream_timeout is None else float(litellm_params.stream_timeout), + ) + logging_obj.record_applied_timeout(applied_timeout) + response: Final = http_handler.response_api_handler( + model=model, + input=with_compaction_items_as_summaries( + with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) + ), + responses_api_provider_config=provider_config, + response_api_optional_request_params=request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=applied_timeout, + _is_async=is_async, + client=client, + fake_stream=fake_stream, + litellm_metadata=litellm_metadata, + shared_session=shared_session, + ) + if isinstance(response, Coroutine): + return _stamped_when_done(response, litellm_metadata, custom_llm_provider) + return _stamped(response, litellm_metadata, custom_llm_provider) + + +def _stamped( + response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator, + litellm_metadata: Mapping[str, object] | None, + custom_llm_provider: str, +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + if not isinstance(response, ResponsesAPIResponse): + return response + stamped: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + responses_api_response=response, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + ) + stamped.hidden_params["custom_llm_provider"] = custom_llm_provider + return stamped + + +async def _stamped_when_done( + response: Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator], + litellm_metadata: Mapping[str, object] | None, + custom_llm_provider: str, +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + return _stamped(await response, litellm_metadata, custom_llm_provider) diff --git a/litellm/native/route_plan.py b/litellm/native/route_plan.py new file mode 100644 index 00000000000..4c9f2f37135 --- /dev/null +++ b/litellm/native/route_plan.py @@ -0,0 +1,165 @@ +"""Which native API format a request reaches its provider in, decided once per request by the client-facing surface.""" + +from dataclasses import dataclass +from typing import Final, TypeAlias + +from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +__all__ = ( + "NativeChat", + "NativeMessages", + "NativeResponses", + "RoutePlan", + "native_messages_config", + "plan_chat_route", + "plan_messages_route", + "plan_responses_route", +) + +_MESSAGES_TO_RESPONSES_PROVIDERS: Final = frozenset({"openai"}) +_PROVIDER_VALUES: Final = frozenset(provider.value for provider in LlmProviders) + + +@dataclass(frozen=True, slots=True) +class NativeMessages: + config: BaseAnthropicMessagesConfig + + +@dataclass(frozen=True, slots=True) +class NativeResponses: + pass + + +@dataclass(frozen=True, slots=True) +class NativeChat: + pass + + +RoutePlan: TypeAlias = NativeMessages | NativeResponses | NativeChat + + +def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: + """Whether the deployment opted into forwarding /v1/messages untranslated. + + The opt-in is ``model_info.supported_endpoints`` containing ``"/v1/messages"``, + declared per deployment in config.yaml and plumbed here as ``kwargs["model_info"]`` + by the router. + """ + if not isinstance(model_info, dict): + return False + supported_endpoints: Final = model_info.get("supported_endpoints") + return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints + + +def _deployment_supports_cache_control_ttl(model_info: object) -> bool: + return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True + + +def native_messages_config( + model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None +) -> BaseAnthropicMessagesConfig | None: + provider_config: Final = ( + ProviderConfigManager.get_provider_anthropic_messages_config( + model=model, + provider=LlmProviders(custom_llm_provider), + ) + if custom_llm_provider is not None and custom_llm_provider in _PROVIDER_VALUES + else None + ) + native_config: Final = ( + provider_config if provider_config is not None and provider_config.serves_api_base(model, api_base) else None + ) + if native_config is not None or not _deployment_passes_through_anthropic_messages(model_info): + return native_config + from litellm.llms.openai_like.messages.transformation import ( + OpenAILikeAnthropicMessagesConfig, + ) + + return OpenAILikeAnthropicMessagesConfig(cache_control_ttl=_deployment_supports_cache_control_ttl(model_info)) + + +def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool: + from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping + + model_info, _ = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=str_keyed_mapping(deployment_model_info), + ) + return model_info.get("mode") == "responses" + + +def _responses_mode_is_lost_by_prefix_strip( + requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object +) -> bool: + """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped. + + ``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a + deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the + chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions. + """ + if requested_model == resolved_model: + return False + return _bridges_to_responses_api( + requested_model, custom_llm_provider, deployment_model_info + ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) + + +def _messages_bridge_to_responses( + custom_llm_provider: str | None, + requested_model: str, + resolved_model: str, + deployment_model_info: object, + force_chat_completions: bool, + stop_requested: bool, +) -> bool: + if force_chat_completions or stop_requested: + return False + if custom_llm_provider in _MESSAGES_TO_RESPONSES_PROVIDERS: + return True + if custom_llm_provider is None: + return False + return _responses_mode_is_lost_by_prefix_strip( + requested_model, resolved_model, custom_llm_provider, deployment_model_info + ) + + +def plan_messages_route( + *, + requested_model: str, + resolved_model: str, + custom_llm_provider: str | None, + model_info: object, + api_base: str | None, + compaction_requested: bool, + force_chat_completions: bool, + stop_requested: bool = False, +) -> RoutePlan: + """``force_chat_completions`` is ``litellm.use_chat_completions_url_for_anthropic_messages``. + + A request with ``stop_sequences`` stays on chat completions. + """ + config: Final = native_messages_config(resolved_model, custom_llm_provider, model_info, api_base) + if config is not None: + return NativeMessages(config) + if not compaction_requested and _messages_bridge_to_responses( + custom_llm_provider, requested_model, resolved_model, model_info, force_chat_completions, stop_requested + ): + return NativeResponses() + return NativeChat() + + +def plan_responses_route( + responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool +) -> NativeResponses | NativeChat: + if responses_api_provider_config is None or use_chat_completions_api: + return NativeChat() + return NativeResponses() + + +def plan_chat_route(responses_api_provider_config: BaseResponsesAPIConfig | None) -> NativeChat | NativeResponses: + """For a deployment ``responses_api_bridge_check`` resolved to mode ``responses``.""" + return NativeChat() if responses_api_provider_config is None else NativeResponses() diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 640a947f9c5..97bd58fd6d1 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -48,7 +48,7 @@ def s3_object_reader(bucket_name: str) -> SyncBucketObjectReader: import boto3 from botocore.credentials import Credentials - from litellm.main import bedrock_converse_chat_completion + from litellm.native.chat_providers.handlers import bedrock_converse_chat_completion credentials: Final[Credentials] = bedrock_converse_chat_completion.get_credentials() s3_client: Final = boto3.client( diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 826de8cfdc9..95956cc4f52 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -69,9 +69,9 @@ def _is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) -def _resolve_openai_api_base(api_base: str | None) -> str: +def resolve_openai_api_base(api_base: str | None) -> str: """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate - and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base + and the ``complete_custom_openai`` chat handler MUST resolve this identically, or a custom base set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate, which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve.""" return ( @@ -243,7 +243,7 @@ def responses_api_bridge_check( # exactly as the chat handler does, so a custom base set via litellm.api_base or # OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and bridged to a /responses route it # lacks. A whitespace-only base collapses to the default too. - resolved_api_base: Final = _resolve_openai_api_base(api_base).strip() + resolved_api_base: Final = resolve_openai_api_base(api_base).strip() on_foundry_openai_endpoint: Final = custom_llm_provider == "azure_ai" and azure_ai_supports_native_responses( model, api_base ) diff --git a/litellm/responses/bridge_executor.py b/litellm/responses/bridge_executor.py new file mode 100644 index 00000000000..2e4d631d93e --- /dev/null +++ b/litellm/responses/bridge_executor.py @@ -0,0 +1,259 @@ +"""Hands a bridged Responses request to the native Responses executor, so the bridge never re-enters `aresponses`.""" + +from collections.abc import Coroutine, Mapping, Sequence +from typing import Final, NoReturn + +import httpx +from aiohttp import ClientSession +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.constants import request_timeout +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.native.responses import execute_responses +from litellm.responses.main import ( + apply_managed_file_id_mapping, + base_llm_http_handler, + reasoning_for_deployment, + requested_reasoning, + resolve_model_provider_for_responses, + resolve_responses_api_provider_config, + with_provider_affinity_header, + with_provider_specific_headers, +) +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import AgenticLoopParams + +__all__ = ("anative_responses", "has_native_responses_config", "native_logging_obj", "native_responses") + +_STRING_KEYED_DICT: Final = TypeAdapter(dict[str, object]) +_CLIENT_HEADERS: Final = TypeAdapter(dict[str, str]) +_STRINGS: Final = TypeAdapter(tuple[str, ...]) + + +def _optional_str(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _string_keyed_dict(value: object) -> dict[str, object] | None: + try: + return _STRING_KEYED_DICT.validate_python(value) if value is not None else None + except ValidationError: + return None + + +def _client_headers(value: object) -> Mapping[str, str] | None: + try: + return _CLIENT_HEADERS.validate_python(value) if isinstance(value, dict) else None + except ValidationError: + return None + + +def _shared_session(value: object) -> ClientSession | None: + return value if isinstance(value, ClientSession) else None + + +def _http_client(value: object) -> HTTPHandler | AsyncHTTPHandler | None: + return value if isinstance(value, (HTTPHandler, AsyncHTTPHandler)) else None + + +def _timeout(value: object) -> float | httpx.Timeout: + return value if isinstance(value, (float, int, httpx.Timeout)) and value else request_timeout + + +def _allowed_openai_params(value: object) -> Sequence[str] | None: + try: + return _STRINGS.validate_python(value) if value is not None else None + except ValidationError: + return None + + +def _responses_target( + model: str, forwarded_kwargs: Mapping[str, object], litellm_params: GenericLiteLLMParams +) -> tuple[str, str, BaseResponsesAPIConfig] | None: + provider_model, custom_llm_provider = resolve_model_provider_for_responses( + model=model, + custom_llm_provider=_optional_str(forwarded_kwargs.get("custom_llm_provider")), + litellm_params=litellm_params, + local_vars={}, + ) + if custom_llm_provider is None: + return None + provider_config: Final = resolve_responses_api_provider_config( + provider_model, custom_llm_provider, forwarded_kwargs.get("model_info"), litellm_params.api_base + ) + return None if provider_config is None else (provider_model, custom_llm_provider, provider_config) + + +def has_native_responses_config(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: + return _responses_target(model, forwarded_kwargs, GenericLiteLLMParams(**forwarded_kwargs)) is not None + + +def _native_responses_call( + request: Mapping[str, object], + model: str, + forwarded_kwargs: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + is_async: bool, +) -> ( + ResponsesAPIResponse + | BaseResponsesAPIStreamingIterator + | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] +): + """Resolve the deployment and hand the translated request to the native Responses executor.""" + logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) + litellm_params: Final = GenericLiteLLMParams(**forwarded_kwargs) + target: Final = _responses_target(model, forwarded_kwargs, litellm_params) + if target is None: + raise ValueError(f"No Responses API config for {model}") + provider_model, custom_llm_provider, provider_config = target + request_input, request_tools = apply_managed_file_id_mapping( + input=request["input"], + tools=request.get("tools"), + kwargs=dict(forwarded_kwargs), + local_vars={}, + ) + params: Final = { + **forwarded_kwargs, + **request, + "input": request_input, + "tools": request_tools, + "custom_llm_provider": custom_llm_provider, + } + deployment_model_info: Final = forwarded_kwargs.get("model_info") + reasoning: Final = reasoning_for_deployment( + requested_reasoning(params, provider_model, custom_llm_provider), + provider_model, + custom_llm_provider, + deployment_model_info, + forwarded_kwargs, + ) + return execute_responses( + http_handler=base_llm_http_handler, + model=provider_model, + input=request_input, + provider_config=provider_config, + requested_params=ResponsesAPIRequestUtils.get_requested_response_api_optional_param( + {key: value for key, value in {**params, "reasoning": reasoning}.items() if key != "reasoning_effort"} + ), + allowed_openai_params=_allowed_openai_params(forwarded_kwargs.get("allowed_openai_params")), + declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (), + deployment_model_info=deployment_model_info, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + user=_optional_str(request.get("user")), + extra_headers=with_provider_affinity_header( + with_provider_specific_headers( + ResponsesAPIRequestUtils.merge_client_forwarded_headers( + extra_headers=_string_keyed_dict(forwarded_kwargs.get("extra_headers")), + client_headers=_client_headers(forwarded_kwargs.get("headers")), + ), + litellm_params, + custom_llm_provider, + ), + litellm_params, + _string_keyed_dict(request.get("metadata")), + forwarded_kwargs, + provider_model, + custom_llm_provider, + ), + extra_body=_string_keyed_dict(forwarded_kwargs.get("extra_body")), + timeout=_timeout(forwarded_kwargs.get("timeout")), + is_async=is_async, + stream=request.get("stream") is True, + client=_http_client(forwarded_kwargs.get("client")), + shared_session=_shared_session(forwarded_kwargs.get("shared_session")), + litellm_metadata=_string_keyed_dict(forwarded_kwargs.get("litellm_metadata")) or {}, + request_kwargs=dict(forwarded_kwargs), + ) + + +_OPENAI_CHAT_COMPLETIONS_MODEL_PREFIX: Final = "openai/chat_completions/" + + +def _requests_chat_completions_api(params: Mapping[str, object]) -> bool: + model: Final = params.get("model") + return bool(params.get("use_chat_completions_api")) or ( + isinstance(model, str) and model.startswith(_OPENAI_CHAT_COMPLETIONS_MODEL_PREFIX) + ) + + +def _reads_responses_cache(params: Mapping[str, object]) -> bool: + return litellm.cache is not None or params.get("caching") is True + + +def native_logging_obj( + logging_obj: LiteLLMLoggingObj | None, forwarded_kwargs: Mapping[str, object], request: Mapping[str, object] +) -> LiteLLMLoggingObj | None: + """The logging object when the native executor can serve the call, None when only `aresponses` can. + + The executor skips the prompt management hooks `aresponses` runs, so a call that needs them re-enters it. + So does a deployment that asks for the chat completions API, which `aresponses` bridges to chat, and a call + that reads the response cache, which `aresponses` keys and fills. + """ + if logging_obj is None: + return None + params: Final = {**forwarded_kwargs, **request} + if _requests_chat_completions_api(params) or _reads_responses_cache(params): + return None + if logging_obj.should_run_prompt_management_hooks( + non_default_params=params, prompt_id=_optional_str(params.get("prompt_id")) + ): + return None + return logging_obj + + +def _raise_mapped(error: Exception, model: str, forwarded_kwargs: Mapping[str, object]) -> NoReturn: + raise litellm.exception_type( + model=model, + custom_llm_provider=_optional_str(forwarded_kwargs.get("custom_llm_provider")), + original_exception=error, + extra_kwargs=dict(forwarded_kwargs), + ) from error + + +def _priced( + response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator, logging_obj: LiteLLMLoggingObj +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + """The response, priced as `aresponses` would have priced it before the bridge translates it.""" + if isinstance(response, ResponsesAPIResponse): + logging_obj.price_bridged_provider_response(response) + else: + logging_obj.price_bridged_provider_stream(response) + return response + + +def native_responses( + request: Mapping[str, object], + model: str, + forwarded_kwargs: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + try: + response: Final = _native_responses_call(request, model, forwarded_kwargs, logging_obj, is_async=False) + except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception aresponses would raise + _raise_mapped(e, model, forwarded_kwargs) + if isinstance(response, Coroutine): + raise TypeError("The sync Responses executor returned a coroutine") + return _priced(response, logging_obj) + + +async def anative_responses( + request: Mapping[str, object], + model: str, + forwarded_kwargs: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, +) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: + try: + response: Final = _native_responses_call(request, model, forwarded_kwargs, logging_obj, is_async=True) + settled: Final = await response if isinstance(response, Coroutine) else response + except Exception as e: # noqa: BLE001 # every provider failure maps to the litellm exception aresponses would raise + _raise_mapped(e, model, forwarded_kwargs) + return _priced(settled, logging_obj) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 1f56ddf863e..13e6c8a9717 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -4,7 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter @@ -30,6 +30,9 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ModelResponse +if TYPE_CHECKING: + from litellm.native.chat_plan import PlannedChat + _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -55,6 +58,29 @@ def _bridged_request( ) +def _plan_bridged_chat(request: Mapping[str, object]) -> "PlannedChat | None": + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.native.chat_plan import plan_chat + + logging_obj: Final = request.get("litellm_logging_obj") + return plan_chat(request, logging_obj) if isinstance(logging_obj, Logging) else None + + +def _price_planned_chat_response( + request: Mapping[str, object], response: ModelResponse | litellm.CustomStreamWrapper +) -> None: + """Prices the chat response as `acompletion` would have, before it becomes a Responses object.""" + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = request.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return + if isinstance(response, ModelResponse): + logging_obj.price_bridged_provider_response(response) + else: + logging_obj.price_bridged_provider_stream(response) + + class LiteLLMCompletionTransformationHandler: def response_api_handler( self, @@ -116,10 +142,18 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) - litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + completion_args: Final = {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True} + + from litellm.native.chat_plan import execute_planned_chat + + planned_chat: Final = _plan_bridged_chat(_REQUEST_KWARGS_ADAPTER.validate_python(completion_args)) + litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = ( + litellm.completion(**completion_args) if planned_chat is None else execute_planned_chat(planned_chat) ) + if planned_chat is not None: + _price_planned_chat_response(completion_args, litellm_completion_response) + if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( @@ -139,6 +173,7 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=bridged_request, custom_llm_provider=custom_llm_provider, litellm_metadata=kwargs.get("litellm_metadata", {}), + logs_completion=planned_chat is not None, ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") @@ -208,9 +243,20 @@ class LiteLLMCompletionTransformationHandler: input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) - litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + acompletion_args: Final = {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True} + + from litellm.native.chat_plan import aexecute_planned_chat + + planned_chat: Final = _plan_bridged_chat(_REQUEST_KWARGS_ADAPTER.validate_python(acompletion_args)) + litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = ( + await litellm.acompletion(**acompletion_args) + if planned_chat is None + else await aexecute_planned_chat(planned_chat) ) + + if planned_chat is not None: + _price_planned_chat_response(acompletion_args, litellm_completion_response) + if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( @@ -230,5 +276,6 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=responses_api_request, custom_llm_provider=litellm_completion_request.get("custom_llm_provider"), litellm_metadata=kwargs.get("litellm_metadata", {}), + logs_completion=planned_chat is not None, ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index ce2396355a2..8f78f57dfe7 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -2,13 +2,16 @@ import asyncio import json from collections.abc import Mapping, Sequence from dataclasses import dataclass -from itertools import chain +from itertools import chain, groupby from typing import TYPE_CHECKING, Any, Final, cast +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.responses.litellm_completion_transformation.client_tool_search import ( bridge_client_tool_search, client_tool_search_loaded_tools, @@ -41,6 +44,7 @@ else: # Cold Storage Handler ######################################################## COLD_STORAGE_HANDLER: Final = ColdStorageHandler() +_STORED_REQUEST: Final = TypeAdapter(dict[str, object]) ######################################################## @@ -181,6 +185,82 @@ def _instruction_items_before( ) +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +def _stored_web_search_results(stored_response: object) -> Sequence[Mapping[str, object]]: + """The Anthropic ``web_search_tool_result`` blocks a bridged response kept in its ``provider_specific_fields``.""" + response: Final = as_str_mapping(stored_response) + fields: Final = None if response is None else as_str_mapping(response.get("provider_specific_fields")) + results: Final = None if fields is None else fields.get("web_search_results") + if not isinstance(results, list): + return () + return tuple(result for result in map(as_str_mapping, _OBJECT_LIST.validate_python(results)) if result is not None) + + +def _server_web_search_call_id(item: object, web_search_results: Sequence[Mapping[str, object]]) -> str | None: + """The ``server_tool_use`` id behind a stored ``web_search_call`` item, when its result block was kept.""" + if not isinstance(item, Mapping) or item.get("type") != "web_search_call": + return None + item_id: Final = item.get("id") + call_id: Final = item_id.removeprefix("ws_") if isinstance(item_id, str) else None + if call_id and any(result.get("tool_use_id") == call_id for result in web_search_results): + return call_id + return None + + +def _server_web_search_message(items: Sequence[object], web_search_results: Sequence[Mapping[str, object]]) -> Message: + """Replay stored web search calls as the assistant tool calls the chat completion carried, + so an Anthropic target rebuilds its ``server_tool_use`` / ``web_search_tool_result`` blocks.""" + calls: Final = tuple( + (call_id, item) + for item in map(as_str_mapping, items) + if item is not None and (call_id := _server_web_search_call_id(item, web_search_results)) is not None + ) + call_ids: Final = frozenset(call_id for call_id, _ in calls) + return Message( + role="assistant", + content=None, + tool_calls=[ + { + "id": call_id, + "type": "function", + "function": {"name": "web_search", "arguments": json.dumps({"query": _web_search_query(item)})}, + } + for call_id, item in calls + ], + provider_specific_fields={ + "web_search_results": [result for result in web_search_results if result.get("tool_use_id") in call_ids] + }, + ) + + +def _web_search_query(item: Mapping[str, object]) -> str: + action: Final = item.get("action") + query: Final = action.get("query") if isinstance(action, Mapping) else None + return query if isinstance(query, str) else "" + + +_CHAT_MESSAGE_CONTENT_ITEM_TYPES: Final = ("reasoning", "message") + + +def _in_chat_message_order( + output: ResponseInputParam, web_search_results: Sequence[Mapping[str, object]] +) -> ResponseInputParam: + """A turn with server web searches in the order its chat completion message replayed it: reasoning, then text, + then the tool calls as they happened. Any other turn keeps its output order.""" + if not any(_server_web_search_call_id(item, web_search_results) is not None for item in output): + return output + return sorted( + output, + key=lambda item: ( + _CHAT_MESSAGE_CONTENT_ITEM_TYPES.index(item_type) + if (item_type := item.get("type")) in _CHAT_MESSAGE_CONTENT_ITEM_TYPES + else len(_CHAT_MESSAGE_CONTENT_ITEM_TYPES) + ), + ) + + def _stored_instructions(proxy_server_request: Mapping[str, object] | None) -> str | None: instructions: Final = None if proxy_server_request is None else proxy_server_request.get("instructions") return instructions if isinstance(instructions, str) and instructions else None @@ -371,15 +451,23 @@ class ResponsesSessionHandler: ############################################################ _response_output: Final[object] = spend_log.get("response", "{}") if (responses_api_output := _stored_responses_api_output(_response_output)) is not None: - chat_completion_message_history.extend( - LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=bridge_client_tool_search(responses_api_output, None).input, - responses_api_request={}, - replay_reasoning=True, - item_cache_control=item_cache_control, - only_since_compaction=True, + web_search_results: Final = _stored_web_search_results(_response_output) + for is_server_web_search, items in groupby( + _in_chat_message_order(responses_api_output, web_search_results), + key=lambda item: _server_web_search_call_id(item, web_search_results) is not None, + ): + if is_server_web_search: + chat_completion_message_history.append(_server_web_search_message(tuple(items), web_search_results)) + continue + chat_completion_message_history.extend( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=bridge_client_tool_search(list(items), None).input, + responses_api_request={}, + replay_reasoning=True, + item_cache_control=item_cache_control, + only_since_compaction=True, + ) ) - ) elif isinstance(_response_output, dict) and _response_output and _response_output != {}: # transform `ChatCompletion Response` to `ResponsesAPIResponse` model_response: Final = ModelResponse(**_response_output) @@ -397,30 +485,23 @@ class ResponsesSessionHandler: Get the parsed proxy server request from the spend log """ proxy_server_request: Final[str | dict] = spend_log.get("proxy_server_request") or "{}" - proxy_server_request_dict: dict[str, object] | None = None - if isinstance(proxy_server_request, dict): - proxy_server_request_dict = proxy_server_request - else: - proxy_server_request_dict = json.loads(proxy_server_request) + proxy_server_request_dict: Final = _STORED_REQUEST.validate_python( + proxy_server_request if isinstance(proxy_server_request, dict) else json.loads(proxy_server_request) + ) ############################################################ # Check if user has setup cold storage for session handling ############################################################ - if ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict): - # Try to get cold storage object key from spend log metadata - _proxy_server_request_dict: dict | None = None - cold_storage_object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) - if cold_storage_object_key: - # Use the object key directly from metadata - _proxy_server_request_dict = ( - await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key( - object_key=cold_storage_object_key, - ) - ) - if _proxy_server_request_dict: - proxy_server_request_dict = _proxy_server_request_dict - - return proxy_server_request_dict + if not ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_server_request_dict): + return proxy_server_request_dict + cold_storage_object_key: Final = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) + if not cold_storage_object_key: + return proxy_server_request_dict + return ( + await ResponsesSessionHandler.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=cold_storage_object_key, + ) + ) or proxy_server_request_dict @staticmethod def _get_cold_storage_object_key_from_spend_log( @@ -462,14 +543,10 @@ class ResponsesSessionHandler: """ verbose_proxy_logger.debug("inside get_proxy_server_request_from_cold_storage_with_object_key...") - proxy_server_request_dict: Final = ( - await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key( - object_key=object_key, - ) + return await COLD_STORAGE_HANDLER.get_proxy_server_request_from_cold_storage_with_object_key( + object_key=object_key, ) - return proxy_server_request_dict - @staticmethod def _should_check_cold_storage_for_full_payload( proxy_server_request_dict: Mapping[str, object] | None, diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 892da541935..a9615841d7f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item from litellm.responses.litellm_completion_transformation.custom_tools import ( @@ -26,7 +27,11 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) -from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost +from litellm.responses.streaming_iterator import ( + ResponsesAPIStreamingIterator, + dispatch_stream_success, + stamp_responses_usage_cost, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PART_UNION_TYPES, @@ -119,6 +124,10 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool: return has_reasoning and bool(delta.content or delta.tool_calls) +def _logged_by_this_stream(_assembled: object) -> None: + return None + + class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): """ Async iterator for processing streaming responses from the Responses API. @@ -132,9 +141,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None = None, litellm_metadata: dict | None = None, + logs_completion: bool = False, ): + """``logs_completion`` makes this stream, not the chat stream it wraps, log the request as a Responses row.""" self.model: str = model self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper + self._logs_completion = logs_completion + if logs_completion: + litellm_custom_stream_wrapper.hand_completion_to(_logged_by_this_stream) self.request_input: str | ResponseInputParam = request_input self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request self.custom_llm_provider: str | None = custom_llm_provider @@ -143,7 +157,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # repeated Pydantic attribute access in end-of-stream assembly. self.collected_chat_completion_chunks: list[dict[str, object]] = [] self.finished: bool = False - self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj + self.litellm_logging_obj: LiteLLMLoggingObj = litellm_custom_stream_wrapper.logging_obj self.sent_response_created_event: bool = False self.sent_response_in_progress_event: bool = False self.sent_output_item_added_event: bool = False @@ -1120,6 +1134,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Latch so wrappers (FallbackResponsesStreamWrapper) + proxy # container-ownership hook can read completed_response. self.completed_response = response_completed_event + if self._logs_completion: + dispatch_stream_success( + self.litellm_logging_obj, + response_completed_event, + start_time=None, + end_time=None, + cache_hit=None, + is_async=not sync_mode, + ) return response_completed_event else: if sync_mode: @@ -1493,6 +1516,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None: self.litellm_custom_stream_wrapper.bill_failure_after_earlier_rounds(earlier) + def log_rounds_as_chat(self) -> None: + """An MCP auto-execute bills its rounds from each chat stream's assembled response, so the chat stream logs.""" + if self._logs_completion: + self._logs_completion = False + self.litellm_custom_stream_wrapper.take_completion_back() + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: """ The response.incomplete event to bill when the client went away mid-stream. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 98356e9a5d4..8d212ada306 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,10 +6,10 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints +from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast, get_type_hints import httpx -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import ( TypeIs, # noqa: TID251 # narrows a client input item before its mark is read assert_never, @@ -29,7 +29,6 @@ from litellm.integrations.anthropic_cache_control_hook import ( configured_injection_points, ) from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -43,6 +42,8 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfi from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig +from litellm.native.responses import execute_responses +from litellm.native.route_plan import NativeChat, plan_responses_route from litellm.responses.litellm_completion_transformation.compaction_items import ( with_compaction_items_as_summaries, ) @@ -92,7 +93,6 @@ from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, AgenticLoopParams, CallTypes, - CostPerToken, all_litellm_params, ) from litellm.utils import ( @@ -565,48 +565,6 @@ async def aresponses_api_with_mcp( return response -class _DeploymentModelInfo(BaseModel): - model_config = ConfigDict(frozen=True) - id: object = None - - -_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter( - Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")] -) - - -def _is_router_deployment(deployment_model_info: object) -> bool: - """A Router stamps the deployment's model_info (with its id) onto every call it routes.""" - parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info) - return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None - - -_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) - - -def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None: - if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None: - return None - return _COST_PER_TOKEN.validate_python( - litellm_params.model_dump( - include={ - "input_cost_per_token", - "output_cost_per_token", - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - }, - exclude_none=True, - ) - ) - - -def _bridges_to_chat_completions( - responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool -) -> bool: - """Whether the request reaches its provider as a chat completion, not a Responses call.""" - return responses_api_provider_config is None or use_chat_completions_api is True - - _RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED: Final = frozenset({"client_metadata"}) @@ -693,7 +651,7 @@ def _encrypted_task_support_failure( ) -> _ResponsesCompatibilityFailure | None: if ( responses_api_provider_config is None - or _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api) + or isinstance(plan_responses_route(responses_api_provider_config, use_chat_completions_api), NativeChat) or not responses_api_provider_config.supports_encrypted_agent_messages() ): return "encrypted_task_unsupported" @@ -737,7 +695,7 @@ class _AsyncPromptManagementOutcome: deployment_model_info: object -def _resolve_responses_api_provider_config( +def resolve_responses_api_provider_config( model: str, custom_llm_provider: str, model_info: object, api_base: str | None ) -> BaseResponsesAPIConfig | None: provider_config: Final = ProviderConfigManager.get_provider_responses_api_config( @@ -770,7 +728,7 @@ def _will_bridge_to_chat_completions( api_base: str | None, litellm_params: GenericLiteLLMParams | None = None, ) -> bool: - """``_bridges_to_chat_completions`` for callers running before the provider config is resolved. + """``plan_responses_route`` for callers running before the provider config is resolved. Resolving the config is a pure lookup, so this asks the same question the dispatch asks rather than restating its condition, with the model name the dispatch hands the @@ -783,14 +741,17 @@ def _will_bridge_to_chat_completions( normalized_model: Final = _normalize_openai_chat_completions_responses_model(model) if custom_llm_provider is None: return True - return _bridges_to_chat_completions( - _resolve_responses_api_provider_config( - _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params), - custom_llm_provider, - model_info, - api_base, + return isinstance( + plan_responses_route( + resolve_responses_api_provider_config( + _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params), + custom_llm_provider, + model_info, + api_base, + ), + use_chat_completions_api or normalized_model[1], ), - use_chat_completions_api or normalized_model[1], + NativeChat, ) @@ -1251,7 +1212,88 @@ def _strip_responses_routing_prefix(model: str) -> str: return model[len(_RESPONSES_ROUTING_PREFIX) :] -def _resolve_model_provider_for_responses( +def with_provider_specific_headers( + extra_headers: dict[str, object] | None, litellm_params: GenericLiteLLMParams, custom_llm_provider: str | None +) -> dict[str, object] | None: + from litellm.litellm_core_utils.core_helpers import attribute_or_none + + return ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=extra_headers, + provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"), + custom_llm_provider=custom_llm_provider, + ) + + +def with_provider_affinity_header( + extra_headers: dict[str, object] | None, + litellm_params: GenericLiteLLMParams, + metadata: Mapping[str, object] | None, + kwargs: Mapping[str, object], + model: str, + custom_llm_provider: str | None, +) -> dict[str, object] | None: + affinity_api_base: Final = ( + get_openai_credentials(api_base=litellm_params.api_base).api_base + if custom_llm_provider == "openai" + else litellm_params.api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) + if litellm_params.provider_affinity_header is None and default_affinity_header is None: + return extra_headers + try: + return add_provider_affinity_header( + headers=extra_headers or MappingProxyType({}), + litellm_params=MappingProxyType( + { + "provider_affinity_header": litellm_params.provider_affinity_header, + "litellm_session_id": kwargs.get("litellm_session_id"), + "session_id": kwargs.get("session_id"), + "metadata": metadata, + "litellm_metadata": kwargs.get("litellm_metadata"), + } + ), + default_header=default_affinity_header, + ) + except ValueError as affinity_error: + raise litellm.BadRequestError( + message=str(affinity_error), model=model, llm_provider=custom_llm_provider + ) from affinity_error + + +def requested_reasoning(params: Mapping[str, object], model: str, custom_llm_provider: str | None) -> Reasoning | None: + current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict + Reasoning | None, params.get("reasoning") + ) + reasoning_effort: Final = params.get("reasoning_effort") + if current_reasoning is None and reasoning_effort is not None: + return LiteLLMResponsesTransformationHandler().map_reasoning_effort( + reasoning_effort, model, custom_llm_provider + ) + return current_reasoning + + +def reasoning_for_deployment( + reasoning: Reasoning | None, + model: str, + custom_llm_provider: str | None, + deployment_model_info: object, + kwargs: Mapping[str, object], +) -> Reasoning | None: + """The requested reasoning with its effort resolved onto the deployment's declared reasoning_effort_levels.""" + return cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given + Reasoning | None, + resolve_deployment_reasoning_effort( + reasoning, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + model_group=model_group_from_metadata(kwargs.get("litellm_metadata")), + drop_params=litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True, + ).value, + ) + + +def resolve_model_provider_for_responses( model: str, custom_llm_provider: str | None, litellm_params: GenericLiteLLMParams, @@ -1276,7 +1318,7 @@ def _resolve_model_provider_for_responses( return _strip_responses_routing_prefix(provider_model), resolved_provider -def _apply_managed_file_id_mapping( +def apply_managed_file_id_mapping( input: str | ResponseInputParam, tools: Iterable[ToolParam] | None, kwargs: dict[str, Any], @@ -1523,7 +1565,6 @@ def responses( try: litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") - litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("aresponses", False) is True skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) require_encrypted_task_support: Final = kwargs.pop("_require_encrypted_task_support", False) is True @@ -1576,44 +1617,12 @@ def responses( ) litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) - from litellm.litellm_core_utils.core_helpers import attribute_or_none - litellm_params: Final = GenericLiteLLMParams(extra_headers=extra_headers, **kwargs) - provider_headers: Final = ResponsesAPIRequestUtils.merge_provider_specific_headers( - extra_headers=extra_headers, - provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"), - custom_llm_provider=custom_llm_provider, - ) + provider_headers: Final = with_provider_specific_headers(extra_headers, litellm_params, custom_llm_provider) local_vars["extra_headers"] = provider_headers - - affinity_api_base: Final = ( - get_openai_credentials(api_base=litellm_params.api_base).api_base - if custom_llm_provider == "openai" - else litellm_params.api_base + effective_extra_headers: Final = with_provider_affinity_header( + provider_headers, litellm_params, metadata, kwargs, model, custom_llm_provider ) - default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) - try: - effective_extra_headers: Final = ( - add_provider_affinity_header( - headers=provider_headers or MappingProxyType({}), - litellm_params=MappingProxyType( - { - "provider_affinity_header": litellm_params.provider_affinity_header, - "litellm_session_id": kwargs.get("litellm_session_id"), - "session_id": kwargs.get("session_id"), - "metadata": metadata, - "litellm_metadata": kwargs.get("litellm_metadata"), - } - ), - default_header=default_affinity_header, - ) - if litellm_params.provider_affinity_header is not None or default_affinity_header is not None - else provider_headers - ) - except ValueError as affinity_error: - raise litellm.BadRequestError( - message=str(affinity_error), model=model, llm_provider=custom_llm_provider - ) from affinity_error ######################################################### # MOCK RESPONSE LOGIC @@ -1621,7 +1630,7 @@ def responses( if litellm_params.mock_response and isinstance(litellm_params.mock_response, str): return mock_responses_api_response(mock_response=litellm_params.mock_response) - model, custom_llm_provider = _resolve_model_provider_for_responses( + model, custom_llm_provider = resolve_model_provider_for_responses( model=model, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, @@ -1631,7 +1640,7 @@ def responses( ######################################################### # Update input and tools with provider-specific file IDs if managed files are used ######################################################### - input, tools = _apply_managed_file_id_mapping(input=input, tools=tools, kwargs=kwargs, local_vars=local_vars) + input, tools = apply_managed_file_id_mapping(input=input, tools=tools, kwargs=kwargs, local_vars=local_vars) ######################################################### # Native MCP Responses API @@ -1674,7 +1683,7 @@ def responses( if custom_llm_provider is None: responses_api_provider_config = None else: - responses_api_provider_config = _resolve_responses_api_provider_config( + responses_api_provider_config = resolve_responses_api_provider_config( model, custom_llm_provider, deployment_model_info, litellm_params.api_base ) @@ -1690,27 +1699,9 @@ def responses( _raise_responses_compatibility_failure(compatibility_failure, model, custom_llm_provider) local_vars.update(kwargs) - current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict - Reasoning | None, local_vars.get("reasoning") - ) - reasoning_effort: Final[object] = local_vars.get("reasoning_effort") - request_reasoning: Final = ( - LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort, model, custom_llm_provider) - if current_reasoning is None and reasoning_effort is not None - else current_reasoning - ) - routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict - deployment_reasoning: Final = cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given - Reasoning | None, - resolve_deployment_reasoning_effort( - request_reasoning, - model=model, - custom_llm_provider=custom_llm_provider, - deployment_model_info=deployment_model_info, - model_group=model_group_from_metadata(routing_kwargs.get("litellm_metadata")), - drop_params=litellm.drop_params is True - or normalize_drop_params(routing_kwargs.get("drop_params")) is True, - ).value, + request_reasoning: Final = requested_reasoning(local_vars, model, custom_llm_provider) + deployment_reasoning: Final = reasoning_for_deployment( + request_reasoning, model, custom_llm_provider, deployment_model_info, kwargs ) response_api_optional_params: Final[ResponsesAPIOptionalRequestParams] = ( ResponsesAPIRequestUtils.get_requested_response_api_optional_param( @@ -1756,7 +1747,7 @@ def responses( if _file_search_dispatch is not None: return _file_search_dispatch - if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api): + if isinstance(plan_responses_route(responses_api_provider_config, use_chat_completions_api), NativeChat): bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params) if isinstance(bridge_kwargs, _UntranslatableResponsesFields): _raise_untranslatable_responses_fields(bridge_kwargs, model, custom_llm_provider) @@ -1775,88 +1766,32 @@ def responses( **bridge_kwargs, ) - # Get optional parameters for the responses API - request_drop_params: Final = kwargs.get("drop_params") - responses_api_request_params: Final[dict] = ResponsesAPIRequestUtils.get_optional_params_responses_api( - model=model, - responses_api_provider_config=responses_api_provider_config, - response_api_optional_params=response_api_optional_params, - allowed_openai_params=allowed_openai_params, - drop_params=normalize_drop_params(request_drop_params), - declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (), - ) - - litellm_logging_obj.update_from_kwargs( - kwargs=kwargs, - model=model, - user=user, - optional_params=dict(responses_api_request_params), - litellm_params={ - **responses_api_request_params, - **litellm_params.custom_pricing_overrides(), - "aresponses": _is_async, - "litellm_call_id": litellm_call_id, - "model_info": kwargs.get("model_info"), - "data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base), - "metadata": (kwargs["litellm_metadata"] if "litellm_metadata" in kwargs else kwargs.get("metadata")), - "provider_affinity_header": litellm_params.provider_affinity_header, - }, - custom_llm_provider=custom_llm_provider, - ) - - if not _is_router_deployment(deployment_model_info): - litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) - litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second - # Decode any litellm-encoded encrypted-content item IDs back to their original IDs - input = with_compaction_items_as_summaries( - with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) - ) - - # Call the handler with _is_async flag instead of directly calling the async handler - if custom_llm_provider is None: + if custom_llm_provider is None or responses_api_provider_config is None: raise ValueError("custom_llm_provider is required but passed as None") - fake_stream: Final = responses_api_provider_config.should_fake_stream( - model=model, stream=stream, custom_llm_provider=custom_llm_provider - ) - applied_timeout: Final = CompletionTimeout.for_stream( - (StreamKind.FAKED if fake_stream else StreamKind.LIVE) if stream is True else StreamKind.NOT_STREAMED, - CompletionTimeout.as_limit(timeout or request_timeout), - None - if stream is not True or litellm_params.stream_timeout is None - else float(litellm_params.stream_timeout), - ) - litellm_logging_obj.record_applied_timeout(applied_timeout) - response = base_llm_http_handler.response_api_handler( + return execute_responses( + http_handler=base_llm_http_handler, model=model, input=input, - responses_api_provider_config=responses_api_provider_config, - response_api_optional_request_params=responses_api_request_params, + provider_config=responses_api_provider_config, + requested_params=response_api_optional_params, + allowed_openai_params=allowed_openai_params, + declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (), + deployment_model_info=deployment_model_info, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, + user=user, extra_headers=effective_extra_headers, extra_body=extra_body, - timeout=applied_timeout, - _is_async=_is_async, + timeout=timeout or request_timeout, + is_async=_is_async, + stream=stream, client=kwargs.get("client"), - fake_stream=fake_stream, - litellm_metadata=kwargs.get("litellm_metadata", {}), shared_session=kwargs.get("shared_session"), + litellm_metadata=kwargs.get("litellm_metadata", {}), + request_kwargs=kwargs, ) - - # Update the responses_api_response_id with the model_id - if isinstance(response, ResponsesAPIResponse): - response = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( - responses_api_response=response, - litellm_metadata=kwargs.get("litellm_metadata", {}), - custom_llm_provider=custom_llm_provider, - ) - # Stamp custom_llm_provider so callbacks can identify the provider - # (mirrors litellm/main.py:1371 for chat completions) - response.hidden_params["custom_llm_provider"] = custom_llm_provider - - return response except Exception as e: raise litellm.exception_type( model=model, @@ -2694,7 +2629,7 @@ def compact_responses( # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) - model, custom_llm_provider = _resolve_model_provider_for_responses( + model, custom_llm_provider = resolve_model_provider_for_responses( model=model, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 189366feab4..739b695d165 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -475,6 +475,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): if not self.should_auto_execute: return None if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator): + self.base_iterator.log_rounds_as_chat() return StreamedRoundBilling(self.base_iterator.litellm_logging_obj) if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator): return StreamedRoundBilling(self.base_iterator.logging_obj) @@ -1046,6 +1047,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._cached_response_id = None self._round_index += 1 if self._round_billing is not None: + if isinstance(follow_up_response, LiteLLMCompletionStreamingIterator): + follow_up_response.log_rounds_as_chat() self._round_billing.record_billed_rounds(follow_up_response) except Exception as e: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6a82304b76a..59e49c44471 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -57,6 +57,7 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_r from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( update_response_metadata, ) +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import ( @@ -366,6 +367,50 @@ def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool: PRE_OUTPUT_LIFECYCLE_EVENT_TYPES: Final = frozenset({"response.created", "response.in_progress", "response.queued"}) +def dispatch_stream_success( + logging_obj: LiteLLMLoggingObj, + logging_response: object, + *, + start_time: datetime | None, + end_time: datetime | None, + cache_hit: bool | None, + is_async: bool, + client_detached: bool = False, +) -> None: + if not is_async: + run_async_function( + async_function=logging_obj.async_success_handler, + result=logging_response, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + ) + executor.submit( + logging_obj.success_handler, + result=logging_response, + cache_hit=cache_hit, + start_time=start_time, + end_time=end_time, + ) + return + logging_coroutine: Final = logging_obj.dispatch_success_handlers( + logging_response, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + prefer_async_handlers=True, + ) + if client_detached: + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) + return + if getattr(logging_obj, "_on_deferred_stream_complete", None) is not None: + # End-of-stream guardrail scans write guardrail_information after + # the terminal event; dispatching now would snapshot metadata early. + logging_obj.hold_stream_completion((logging_coroutine,)) + return + spawn_logging_task(asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging") + + class BaseResponsesAPIStreamingIterator: """ Base class for streaming iterators that process responses from the Responses API. @@ -377,6 +422,7 @@ class BaseResponsesAPIStreamingIterator: _started_tool_call: bool = False _earlier_rounds_spend: BilledRoundsSpend | None = None _replays_stored_stream: bool = False + _on_complete: Callable[[object], None] | None = None def __init__( self, @@ -430,6 +476,10 @@ class BaseResponsesAPIStreamingIterator: ) # GUARANTEE OPENAI HEADERS IN RESPONSE self._raw_response_headers: Mapping[str, str] = MappingProxyType(dict(self.response.headers or {})) + def hand_completion_to(self, on_complete: Callable[[object], None]) -> None: + """A bridge whose own client stream logs the request takes the completed response instead of the logger.""" + self._on_complete = on_complete + def _check_max_streaming_duration(self) -> None: """Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS.""" if LITELLM_MAX_STREAMING_DURATION_SECONDS is None: @@ -634,37 +684,16 @@ class BaseResponsesAPIStreamingIterator: self._restore_provider_response_headers(logging_response) end_time: Final = datetime.now() - if is_async: - logging_coroutine: Final = self.logging_obj.dispatch_success_handlers( + if self._on_complete is not None: + self._on_complete(logging_response) + else: + dispatch_stream_success( + self.logging_obj, logging_response, start_time=self.start_time, end_time=end_time, cache_hit=self._completed_response_cache_hit, - prefer_async_handlers=True, - ) - deferred_dispatch_armed: Final = getattr(self.logging_obj, "_on_deferred_stream_complete", None) is not None - if deferred_dispatch_armed: - # End-of-stream guardrail scans write guardrail_information after - # the terminal event; dispatching now would snapshot metadata early. - self.logging_obj._deferred_stream_complete_args = (logging_coroutine,) - else: - spawn_logging_task( - asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging" - ) - else: - run_async_function( - async_function=self.logging_obj.async_success_handler, - result=logging_response, - start_time=self.start_time, - end_time=end_time, - cache_hit=self._completed_response_cache_hit, - ) - executor.submit( - self.logging_obj.success_handler, - result=logging_response, - cache_hit=self._completed_response_cache_hit, - start_time=self.start_time, - end_time=end_time, + is_async=is_async, ) self._run_post_success_hooks(end_time=end_time) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index b0db0516658..2424c0f0862 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -49,7 +49,9 @@ class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): pass -def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]: +def _apply_nested_drop_params( + params: dict[str, object], additional_drop_params: Sequence[str] | None +) -> dict[str, object]: nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path)) return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params) @@ -224,7 +226,7 @@ class ResponsesAPIRequestUtils: @staticmethod def merge_client_forwarded_headers( extra_headers: dict[str, object] | None, - client_headers: dict[str, str] | None, + client_headers: Mapping[str, str] | None, ) -> dict[str, object] | None: """ Merge headers forwarded by the proxy (`headers` kwarg, set when @@ -259,8 +261,8 @@ class ResponsesAPIRequestUtils: @staticmethod def _check_valid_arg( - supported_params: list[str] | None, - non_default_params: dict, + supported_params: Sequence[str] | None, + non_default_params: Mapping[str, object], drop_params: bool | None, custom_llm_provider: str | None, model: str, @@ -285,7 +287,7 @@ class ResponsesAPIRequestUtils: model: str, responses_api_provider_config: BaseResponsesAPIConfig, response_api_optional_params: ResponsesAPIOptionalRequestParams, - allowed_openai_params: list[str] | None = None, + allowed_openai_params: Sequence[str] | None = None, drop_params: bool | None = None, declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: @@ -311,7 +313,7 @@ class ResponsesAPIRequestUtils: non_default_params: Final = cast(dict, response_api_optional_params) # Check for unsupported parameters ResponsesAPIRequestUtils._check_valid_arg( - supported_params=supported_params + (allowed_openai_params or []), + supported_params=[*supported_params, *(allowed_openai_params or ())], non_default_params=non_default_params, drop_params=should_drop_params, custom_llm_provider=responses_api_provider_config.custom_llm_provider, @@ -340,7 +342,7 @@ class ResponsesAPIRequestUtils: return apply_openai_param_overrides( optional_params=params_with_normalized_stream_options, non_default_params=non_default_params, - allowed_openai_params=allowed_openai_params or [], + allowed_openai_params=list(allowed_openai_params or ()), ) @staticmethod @@ -1311,7 +1313,7 @@ class ResponseAPILoggingUtils: output_tokens: Final = usage_input.get("output_tokens") if isinstance(input_tokens, int) and isinstance(output_tokens, int): usage_input["total_tokens"] = input_tokens + output_tokens - response_api_usage = ResponseAPIUsage(**usage_input) + response_api_usage = ResponseAPIUsage.model_validate(usage_input) else: response_api_usage = usage_input prompt_tokens: Final[int] = response_api_usage.input_tokens or 0 diff --git a/litellm/types/completion.py b/litellm/types/completion.py index 0f4bdbd75ce..bc8456fb6ad 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -245,7 +245,7 @@ class CompletionDispatchContext: _CompletionDispatchContext = CompletionDispatchContext -_CompletionDispatchResult = Union[ +CompletionDispatchResult = Union[ Coroutine[object, object, Union["ModelResponse", "CustomStreamWrapper"]], "ModelResponse", "CustomStreamWrapper", diff --git a/litellm/utils.py b/litellm/utils.py index 549590e1cf5..f4958b1ddc3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9251,11 +9251,11 @@ def validate_and_fix_openai_messages(messages: list): return validate_chat_completion_user_messages(messages=new_messages) -def validate_and_fix_openai_tools(tools: list | None) -> list[dict] | None: +def validate_and_fix_openai_tools(tools: list | None) -> list[dict[str, object]] | None: """ Ensure tools is List[dict] and not List[BaseModel] """ - new_tools: Final = [] + new_tools: Final[list[dict[str, object]]] = [] if tools is None: return tools for tool in tools: @@ -9343,9 +9343,9 @@ def validate_chat_completion_user_messages(messages: list[AllMessageValues]): def validate_chat_completion_tool_choice( - tool_choice: dict | str | None, + tool_choice: dict[str, object] | str | None, model: str = "", -) -> dict | str | None: +) -> dict[str, object] | str | None: """ Confirm the tool choice is passed in the OpenAI format. @@ -9376,7 +9376,7 @@ def validate_chat_completion_tool_choice( ) -def validate_openai_optional_params(stop: str | list[str] | None = None, **kwargs) -> str | list[str] | None: +def validate_openai_optional_params(stop: str | list[str] | None = None, **kwargs: object) -> str | list[str] | None: """ Validates and fixes OpenAI optional parameters. @@ -11426,8 +11426,8 @@ def reasoning_effort_with_summary_alias( def strip_reasoning_summary_aliases_from_optional_params( - optional_params: dict, -) -> tuple[dict, object | None]: + optional_params: Mapping[str, object], +) -> tuple[dict[str, object], object | None]: """Copy optional_params; remove reasoningSummary aliases from top-level and extra_body.""" op: Final = dict(optional_params) rs_val = op.pop("reasoningSummary", None) diff --git a/scripts/check_native_imports.py b/scripts/check_native_imports.py new file mode 100644 index 00000000000..f40bad0e430 --- /dev/null +++ b/scripts/check_native_imports.py @@ -0,0 +1,58 @@ +import ast +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +PACKAGE: Final = Path("litellm/native") +FORBIDDEN: Final = "litellm.main" + + +def _is_forbidden(module: str) -> bool: + return module == FORBIDDEN or module.startswith(f"{FORBIDDEN}.") + + +def _from_import_targets(package: tuple[str, ...], node: ast.ImportFrom) -> Iterator[str]: + base: Final = package[: len(package) - node.level + 1] if node.level else () + module: Final = ".".join((*base, node.module) if node.module else base) + yield module + yield from (f"{module}.{alias.name}" for alias in node.names) + + +def _imported(package: tuple[str, ...], node: ast.AST) -> Iterator[str]: + match node: + case ast.Import(names=names): + yield from (alias.name for alias in names) + case ast.ImportFrom(): + yield from _from_import_targets(package, node) + case ast.Attribute(value=ast.Name(id="litellm"), attr=attr): + yield f"litellm.{attr}" + case _: + return + + +def violations(path: Path, source: str) -> tuple[str, ...]: + package: Final = ( + ("litellm", "native", *path.relative_to(PACKAGE).parent.parts) if path.is_relative_to(PACKAGE) else () + ) + tree: Final = ast.parse(source, filename=str(path)) + return tuple( + f"{path}:{node.lineno}: litellm.native must not depend on litellm.main" + for node in ast.walk(tree) + if any(_is_forbidden(target) for target in _imported(package, node)) + ) + + +def main() -> int: + findings: Final = tuple( + finding for path in sorted(PACKAGE.rglob("*.py")) for finding in violations(path, path.read_text()) + ) + if findings: + sys.stderr.write("\n".join(findings) + "\n") + return 1 + sys.stdout.write("litellm.native imports: passed\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 95b5942fa8b..d07bd8705e7 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -107,6 +107,9 @@ litellm_py_files=$(scope_match "$litellm_py_pattern") if [ -n "$(scope_match '^(litellm/proxy/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1 fi +if [ -n "$(scope_match '^(litellm/native/|scripts/check_native_imports\.py)')" ]; then + uv run --no-sync python scripts/check_native_imports.py || exit 1 +fi e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index c5d7ca96a21..6995998899a 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -1,15 +1,26 @@ +import asyncio +import json +import math +from collections.abc import Mapping, Sequence from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest - +import respx +from openai.types.chat import ChatCompletion +from pydantic import TypeAdapter import litellm +from litellm.caching.caching import Cache from litellm.completion_extras.litellm_responses_transformation.handler import ( ResponsesToCompletionBridgeHandler, ) +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.responses import bridge_executor from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse @@ -41,6 +52,11 @@ def test_is_preformatted_cached_chat_stream_false_wrong_type(): ) +@pytest.fixture +def _cache_on(monkeypatch): + monkeypatch.setattr(litellm, "cache", MagicMock(spec=Cache)) + + def _bridge_kwargs(stream: bool): logging_obj = LiteLLMLogging( litellm_call_id="test-call", @@ -63,7 +79,7 @@ def _bridge_kwargs(stream: bool): } -def test_completion_returns_cached_model_response_directly(): +def test_completion_returns_cached_model_response_directly(_cache_on): """Non-streaming bridge cache hit: responses() returns a ModelResponse -> bridge returns it as-is.""" cached = ModelResponse(id="chatcmpl-cached-nonstream", model="gpt-5.4") bridge = ResponsesToCompletionBridgeHandler() @@ -82,7 +98,7 @@ def test_completion_returns_cached_model_response_directly(): @pytest.mark.asyncio -async def test_acompletion_returns_cached_model_response_directly(): +async def test_acompletion_returns_cached_model_response_directly(_cache_on): cached = ModelResponse(id="chatcmpl-cached-nonstream-async", model="gpt-5.4") bridge = ResponsesToCompletionBridgeHandler() @@ -99,7 +115,7 @@ async def test_acompletion_returns_cached_model_response_directly(): assert result is cached -def test_completion_skips_rewrapping_preformatted_cached_chat_stream(): +def test_completion_skips_rewrapping_preformatted_cached_chat_stream(_cache_on): """Streaming bridge cache hit returning CustomStreamWrapper(cached_response) -> bridge skips re-wrapping.""" stream = MagicMock(spec=CustomStreamWrapper) stream.custom_llm_provider = "cached_response" @@ -124,7 +140,7 @@ def test_completion_skips_rewrapping_preformatted_cached_chat_stream(): assert result is stream -def test_completion_preserves_top_level_stream_flag_in_responses_request(): +def test_completion_preserves_top_level_stream_flag_in_responses_request(_cache_on): stream = MagicMock(spec=CustomStreamWrapper) stream.custom_llm_provider = "cached_response" bridge = ResponsesToCompletionBridgeHandler() @@ -152,7 +168,7 @@ def test_completion_preserves_top_level_stream_flag_in_responses_request(): @pytest.mark.asyncio -async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(): +async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(_cache_on): stream = MagicMock(spec=CustomStreamWrapper) stream.custom_llm_provider = "cached_response" bridge = ResponsesToCompletionBridgeHandler() @@ -177,7 +193,7 @@ async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(): @pytest.mark.asyncio -async def test_acompletion_preserves_top_level_stream_flag_in_responses_request(): +async def test_acompletion_preserves_top_level_stream_flag_in_responses_request(_cache_on): stream = MagicMock(spec=CustomStreamWrapper) stream.custom_llm_provider = "cached_response" bridge = ResponsesToCompletionBridgeHandler() @@ -219,7 +235,7 @@ def _completed_chat_response() -> ModelResponse: @pytest.mark.asyncio -async def test_acompletion_streams_completed_model_response(): +async def test_acompletion_streams_completed_model_response(_cache_on): """A streaming request whose bridge call comes back already completed must still be handed back as an async-iterable stream. Returning the bare ModelResponse crashed the proxy's SSE generator with "'async for' requires an object with __aiter__ method". @@ -245,7 +261,7 @@ async def test_acompletion_streams_completed_model_response(): assert [c for c in chunks if c.choices[0].finish_reason], "stream never emitted a finish_reason" -def test_completion_streams_completed_model_response(): +def test_completion_streams_completed_model_response(_cache_on): completed = _completed_chat_response() bridge = ResponsesToCompletionBridgeHandler() @@ -288,7 +304,7 @@ def _upstream_model_for(handed_model: str, custom_llm_provider: str) -> str: def test_completion_keeps_provider_native_model_id_through_responses( custom_llm_provider, bridge_model, expected_upstream_model ): - """responses() resolves the provider itself, so the bridge must not hand it an already-stripped model.""" + """The Responses executor resolves the provider itself, so the bridge must not hand it an already-stripped model.""" cached = ModelResponse(id="chatcmpl-cached", model=bridge_model) bridge = ResponsesToCompletionBridgeHandler() kwargs = _bridge_kwargs(stream=False) @@ -301,11 +317,11 @@ def test_completion_keeps_provider_native_model_id_through_responses( "transform_request", return_value={"model": bridge_model, "input": "hi"}, ), - patch("litellm.responses", return_value=cached) as responses_call, + patch.object(bridge_executor, "native_responses", return_value=cached) as responses_call, ): bridge.completion(**kwargs) - handed_model = responses_call.call_args.kwargs["model"] + handed_model = responses_call.call_args.args[1] assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model @@ -329,9 +345,306 @@ async def test_acompletion_keeps_provider_native_model_id_through_responses( "transform_request", return_value={"model": bridge_model, "input": "hi"}, ), - patch("litellm.aresponses", new=AsyncMock(return_value=cached)) as responses_call, + patch.object(bridge_executor, "anative_responses", new=AsyncMock(return_value=cached)) as responses_call, ): await bridge.acompletion(**kwargs) - handed_model = responses_call.call_args.kwargs["model"] + handed_model = responses_call.call_args.args[1] assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model + + +class _DeploymentHookCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def async_pre_call_deployment_hook(self, kwargs, call_type) -> None: + self.calls += 1 + + +@pytest.mark.asyncio +async def test_bridged_chat_call_runs_deployment_hooks_once(monkeypatch): + counter = _DeploymentHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + litellm.register_model( + {"openai/bridge-only-model": {"litellm_provider": "openai", "mode": "responses", "max_tokens": 100}} + ) + + with respx.mock(assert_all_called=False) as upstream: + responses_route = upstream.post("https://example.invalid/v1/responses").mock( + return_value=httpx.Response( + 200, + json={ + "id": "resp_bridge", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "bridge-only-model", + "output": [ + { + "type": "message", + "id": "msg_bridge", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "from upstream", "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + }, + ) + ) + response = await litellm.acompletion( + model="openai/bridge-only-model", + api_base="https://example.invalid/v1", + api_key="x", + messages=[{"role": "user", "content": "hi"}], + ) + + assert response.choices[0].message.content == "from upstream" + assert responses_route.call_count == 1 + assert counter.calls == 1 + + +@pytest.mark.parametrize( + "bridge_model, tools", + [ + ("gpt-5.4", [{"type": "mcp", "server_label": "docs", "server_url": "litellm_proxy"}]), + ("gpt-5.4", [{"type": "file_search", "vector_store_ids": ["vs_1"]}]), + ("chat_completions/gpt-5.4", None), + ], +) +def test_completion_keeps_responses_surface_for_what_only_it_serves(bridge_model, tools): + cached = ModelResponse(id="chatcmpl-surface", model="gpt-5.4") + bridge = ResponsesToCompletionBridgeHandler() + kwargs = _bridge_kwargs(stream=False) + kwargs["model"] = bridge_model + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": bridge_model, "input": "hi", "tools": tools}, + ), + patch("litellm.responses", return_value=cached) as responses_call, + patch.object(bridge_executor, "native_responses") as executor_call, + ): + result = bridge.completion(**kwargs) + + assert result is cached + assert responses_call.call_count == 1 + assert executor_call.call_count == 0 + + +_LOGGED_ROW: Final = TypeAdapter(dict[str, object]) +_LOGGED_CHAT: Final = TypeAdapter(ChatCompletion) + + +class _SuccessRowRecorder(CustomLogger): + rows: tuple[dict[str, object], ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.rows = (*self.rows, _LOGGED_ROW.validate_python(kwargs["standard_logging_object"])) + + +def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +@pytest.mark.asyncio +async def test_bridged_chat_stream_logs_one_chat_row_billed_from_provider_usage(monkeypatch: pytest.MonkeyPatch): + recorder: Final = _SuccessRowRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, "cache_read_input_token_cost": 1e-7} + litellm.register_model( + { + "openai/bridge-stream-model": { + "litellm_provider": "openai", + "mode": "responses", + "max_tokens": 100, + **prices, + } + } + ) + usage: Final[dict[str, object]] = { + "input_tokens": 100, + "input_tokens_details": {"cached_tokens": 40}, + "output_tokens": 20, + "output_tokens_details": {"reasoning_tokens": 5}, + "total_tokens": 120, + } + message: Final[dict[str, object]] = { + "type": "message", + "id": "msg_stream", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "streamed", "annotations": []}], + } + response: Final[dict[str, object]] = { + "id": "resp_stream", + "object": "response", + "created_at": 1, + "model": "bridge-stream-model", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + sse: Final = _responses_sse( + [ + {"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}}, + {"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": message}, + { + "type": "response.output_text.delta", + "sequence_number": 2, + "item_id": "msg_stream", + "output_index": 0, + "content_index": 0, + "delta": "streamed", + }, + {"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": message}, + { + "type": "response.completed", + "sequence_number": 4, + "response": {**response, "status": "completed", "output": [message], "usage": usage}, + }, + ] + ) + + with respx.mock(assert_all_called=False) as upstream: + responses_route = upstream.post("https://example.invalid/v1/responses").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse) + ) + stream = await litellm.acompletion( + model="openai/bridge-stream-model", + api_base="https://example.invalid/v1", + api_key="x", + messages=[{"role": "user", "content": "hi"}], + stream=True, + ) + assert isinstance(stream, CustomStreamWrapper) + text = "".join([chunk.choices[0].delta.content or "" async for chunk in stream if chunk.choices]) + for _ in range(100): + if recorder.rows: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.1) + + assert text == "streamed" + assert responses_route.call_count == 1 + assert len(recorder.rows) == 1 + row: Final = recorder.rows[0] + logged: Final = _LOGGED_CHAT.validate_python(row["response"]) + assert logged.object == "chat.completion" + assert logged.choices[0].message.content == "streamed" + assert row["prompt_tokens"] == 100 + assert row["completion_tokens"] == 20 + cost: Final = row["response_cost"] + assert isinstance(cost, float) + assert math.isclose( + cost, + 60 * prices["input_cost_per_token"] + + 40 * prices["cache_read_input_token_cost"] + + 20 * prices["output_cost_per_token"], + ) + + +_BUILT_IN_TOOL_CALLS: Final[Mapping[str, tuple[Sequence[Mapping[str, object]], Sequence[Mapping[str, object]]]]] = { + "web_search": ( + tuple( + { + "type": "web_search_call", + "id": f"ws_{i}", + "status": "completed", + "action": {"type": "search", "query": "q"}, + } + for i in range(3) + ), + (), + ), + "image_generation": ( + ( + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "result": "aGk=", + "quality": "low", + "size": "1024x1024", + }, + ), + ({"type": "image_generation", "model": "gpt-image-1"},), + ), +} + + +def _responses_body_with(tool_call: str) -> dict[str, object]: + output_items, tools = _BUILT_IN_TOOL_CALLS[tool_call] + message: Final[dict[str, object]] = { + "type": "message", + "id": "msg_tool", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "done", "annotations": []}], + } + return { + "id": "resp_tool", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4.1", + "output": [*output_items, message], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": list(tools), + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +async def _rows_for(recorder: _SuccessRowRecorder, count: int) -> tuple[dict[str, object], ...]: + for _ in range(200): + if len(recorder.rows) >= count: + break + await asyncio.sleep(0.01) + return recorder.rows + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tool_call", sorted(_BUILT_IN_TOOL_CALLS)) +async def test_bridged_chat_call_is_billed_for_the_built_in_tool_calls_the_provider_made( + monkeypatch: pytest.MonkeyPatch, tool_call: str +): + recorder: Final = _SuccessRowRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://example.invalid/v1/responses").mock( + return_value=httpx.Response(200, json=_responses_body_with(tool_call)) + ) + bridged = await litellm.acompletion( + model="openai/responses/gpt-4.1", + api_base="https://example.invalid/v1", + api_key="x", + messages=[{"role": "user", "content": "hi"}], + ) + await _rows_for(recorder, 1) + await litellm.aresponses(model="openai/gpt-4.1", api_base="https://example.invalid/v1", api_key="x", input="hi") + rows: Final = await _rows_for(recorder, 2) + + bridged_row, native_row = rows + assert bridged_row["call_type"] == "acompletion" + assert native_row["call_type"] == "aresponses" + native_cost: Final = native_row["response_cost"] + assert isinstance(native_cost, float) + assert bridged_row["response_cost"] == pytest.approx(native_cost) + assert bridged._hidden_params["response_cost"] == pytest.approx(native_cost) + breakdown: Final = _LOGGED_ROW.validate_python(bridged_row["cost_breakdown"]) + tool_usage_cost: Final = breakdown["tool_usage_cost"] + assert isinstance(tool_usage_cost, float) and tool_usage_cost > 0 diff --git a/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py b/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py index 5aff9397747..210ec951620 100644 --- a/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py +++ b/tests/unit/completion_extras/test_responses_bridge_provider_propagation.py @@ -13,6 +13,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import ( ResponsesToCompletionBridgeHandler, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.responses import bridge_executor from litellm.types.utils import ModelResponse MODEL = "openai.gpt-5.5" @@ -73,11 +74,11 @@ def test_completion_forwards_custom_llm_provider_to_responses(): bridge = ResponsesToCompletionBridgeHandler() cached = ModelResponse(id="chatcmpl-cached", model="gpt-5.5") - with patch("litellm.responses", return_value=cached) as fake_responses: + with patch.object(bridge_executor, "native_responses", return_value=cached) as fake_responses: result = bridge.completion(**_openai_kwargs()) assert result is cached - assert fake_responses.call_args.kwargs["custom_llm_provider"] == "openai" + assert fake_responses.call_args.args[2]["custom_llm_provider"] == "openai" @pytest.mark.asyncio @@ -85,13 +86,13 @@ async def test_acompletion_forwards_custom_llm_provider_to_aresponses(): bridge = ResponsesToCompletionBridgeHandler() cached = ModelResponse(id="chatcmpl-cached", model="gpt-5.5") - async def _fake_aresponses(**kwargs): - _fake_aresponses.kwargs = kwargs + async def _fake_aresponses(request, model, forwarded_kwargs, logging_obj): + _fake_aresponses.kwargs = forwarded_kwargs return cached _fake_aresponses.kwargs = {} - with patch("litellm.aresponses", _fake_aresponses): + with patch.object(bridge_executor, "anative_responses", _fake_aresponses): result = await bridge.acompletion(**_openai_kwargs()) assert result is cached @@ -103,13 +104,13 @@ async def test_acompletion_forwards_aws_region_name_to_aresponses(): bridge = ResponsesToCompletionBridgeHandler() cached = ModelResponse(id="chatcmpl-cached", model=MODEL) - async def _fake_aresponses(**kwargs): - _fake_aresponses.kwargs = kwargs + async def _fake_aresponses(request, model, forwarded_kwargs, logging_obj): + _fake_aresponses.kwargs = forwarded_kwargs return cached _fake_aresponses.kwargs = {} - with patch("litellm.aresponses", _fake_aresponses): + with patch.object(bridge_executor, "anative_responses", _fake_aresponses): result = await bridge.acompletion(**_bedrock_mantle_kwargs()) assert result is cached diff --git a/tests/unit/litellm_core_utils/test_get_litellm_params.py b/tests/unit/litellm_core_utils/test_get_litellm_params.py index 738ab774b2f..a023d28f3da 100644 --- a/tests/unit/litellm_core_utils/test_get_litellm_params.py +++ b/tests/unit/litellm_core_utils/test_get_litellm_params.py @@ -31,11 +31,11 @@ def _funnel_kwargs_completion_forwards(monkeypatch, wif_kwargs: dict[str, object from unittest.mock import MagicMock import litellm - import litellm.main as litellm_main + import litellm.native.chat as native_chat - spy = MagicMock(wraps=litellm_main.get_litellm_params) + spy = MagicMock(wraps=native_chat.get_litellm_params) monkeypatch.setattr( # test-quality-ok: completion() has no injection seam for its kwargs funnel - litellm_main, "get_litellm_params", spy + native_chat, "get_litellm_params", spy ) litellm.completion( model="anthropic/claude-sonnet-5", diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index c30644e7fdc..3d6e4bb8a9f 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -66,6 +66,14 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None: assert result["usage"]["input_tokens"] == 10 +def test_translate_openai_response_to_anthropic_without_usage_reports_zero_tokens() -> None: + response: Final = ModelResponse(choices=[Choices(message=Message(content="x"), finish_reason="stop")]) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert (result["usage"]["input_tokens"], result["usage"]["output_tokens"]) == (0, 0) + + @pytest.mark.parametrize("text,count,expected_stop", [ ("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"), (" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"), diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_chat_executor.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_chat_executor.py new file mode 100644 index 00000000000..db3e380da8e --- /dev/null +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_chat_executor.py @@ -0,0 +1,380 @@ +import asyncio +import datetime +from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from types import MappingProxyType +from typing import Final, TypeAlias, cast + +import httpx +import pytest +import respx +from pydantic import TypeAdapter + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_prompt_management import CustomPromptManagement +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream +from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec +from litellm.types.utils import StandardCallbackDynamicParams + +API_BASE: Final = "http://chat-upstream.test/v1" +CHAT_RESPONSE: Final = { + "id": "chatcmpl-bridge", + "object": "chat.completion", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "from upstream"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, +} + + +class _DeploymentHookCounter(CustomLogger): + pre_call: int = 0 + post_call: int = 0 + + async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None: + self.pre_call += 1 + + async def async_post_call_success_deployment_hook( + self, request_data: Mapping[str, object], response: object, call_type: object + ) -> None: + self.post_call += 1 + + +_PromptResult: TypeAlias = tuple[str, list[AllMessageValues], dict[str, object]] + + +class _TemplatePromptManager(CustomPromptManagement): + async def async_get_chat_completion_prompt( + self, + model: str, + messages: Sequence[AllMessageValues], + non_default_params: Mapping[str, object], + prompt_id: str | None, + prompt_variables: Mapping[str, object] | None, + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: Logging, + prompt_spec: PromptSpec | None = None, + tools: Sequence[Mapping[str, object]] | None = None, + prompt_label: str | None = None, + prompt_version: int | None = None, + ignore_prompt_manager_model: bool | None = False, + ignore_prompt_manager_optional_params: bool | None = False, + ) -> _PromptResult: + return model, [{"role": "user", "content": "from the template"}], {**non_default_params} + + +@pytest.fixture +def upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + return respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=CHAT_RESPONSE)) + + +async def _ask(extra: Mapping[str, object] = MappingProxyType({})) -> object: + return await litellm.anthropic.messages.acreate( + model="hosted_vllm/served-model", + api_base=API_BASE, + api_key="sk-chat", + max_tokens=10, + messages=[{"role": "user", "content": "hi"}], + **extra, + ) + + +@pytest.mark.asyncio +async def test_bridge_runs_deployment_hooks_once_per_request(upstream: respx.Route, monkeypatch: pytest.MonkeyPatch): + counter: Final = _DeploymentHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + + response: Final = await _ask() + + assert isinstance(response, dict) + assert response["content"] == [{"type": "text", "text": "from upstream"}] + assert upstream.call_count == 1 + assert (counter.pre_call, counter.post_call) == (1, 1) + + +@pytest.mark.asyncio +async def test_bridge_streams_the_upstream_reply(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + chunk: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":"streamed"},"finish_reason":null}]}' + done: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}' + respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, + content=f"data: {chunk}\n\ndata: {done}\n\ndata: [DONE]\n\n".encode(), + headers={"content-type": "text/event-stream"}, + ) + ) + + stream: Final = await _ask({"stream": True}) + body: Final = b"".join([event async for event in stream]).decode() + + assert "event: message_start" in body + assert '"text":"streamed"' in body.replace(" ", "") + + +@pytest.mark.asyncio +async def test_bridge_applies_the_prompt_template_named_by_prompt_id( + upstream: respx.Route, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()]) + + await _ask({"prompt_id": "bridge_template"}) + + assert "from the template" in upstream.calls.last.request.content.decode() + + +@pytest.mark.asyncio +async def test_bridge_maps_an_upstream_429_to_rate_limit_error( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit"}}) + ) + + with pytest.raises(litellm.RateLimitError): + await _ask() + + +def _sync_bridge(logging_obj: Logging, *, stream: bool) -> object: + return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + max_tokens=10, + messages=[{"role": "user", "content": "hi"}], + model="hosted_vllm/served-model", + stream=stream, + api_base=API_BASE, + api_key="sk-chat", + litellm_logging_obj=logging_obj, + ) + + +def test_sync_bridge_answers_from_one_upstream_call(upstream: respx.Route): + logging_obj: Final = Logging( + model="hosted_vllm/served-model", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="6825beef-0000-4000-8000-000000000005", + function_id="1234", + ) + + response: Final = _sync_bridge(logging_obj, stream=False) + + assert isinstance(response, dict) + assert response["content"] == [{"type": "text", "text": "from upstream"}] + assert upstream.call_count == 1 + + +@pytest.mark.asyncio +async def test_bridge_runs_prompt_management_triggered_without_prompt_id( + upstream: respx.Route, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()]) + + await _ask({"vector_store_ids": ["vs_bridge"]}) + + assert "from the template" in upstream.calls.last.request.content.decode() + + +@pytest.mark.asyncio +async def test_bridge_keeps_the_litellm_system_prompt(upstream: respx.Route): + await _ask({"litellm_system_prompt": "answer like a pirate"}) + + assert "answer like a pirate" in upstream.calls.last.request.content.decode() + + +@pytest.mark.asyncio +async def test_bridge_goes_through_a_wrapped_acompletion(upstream: respx.Route, monkeypatch: pytest.MonkeyPatch): + unwrapped: Final = litellm.acompletion + seen: Final[list[object]] = [] # mutable-ok: records the calls the wrapper sees + + async def traced_acompletion(*args: object, **kwargs: object) -> object: + seen.append(kwargs.get("model")) + return await unwrapped(*args, **kwargs) + + monkeypatch.setattr(litellm, "acompletion", traced_acompletion) + + response: Final = await _ask() + + assert isinstance(response, dict) + assert response["content"] == [{"type": "text", "text": "from upstream"}] + assert seen == ["hosted_vllm/served-model"] + assert upstream.call_count == 1 + + +class _SuccessRowRecorder(CustomLogger): + call_types: tuple[str, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.call_types = (*self.call_types, str(kwargs.get("call_type"))) + + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.call_types = (*self.call_types, str(kwargs.get("call_type"))) + + +_PARKED_COMPLETION: Final = TypeAdapter(tuple[object]) + + +async def _release(parked: Coroutine[object, object, object]) -> None: + await parked + + +async def _settled_rows(recorder: _SuccessRowRecorder) -> tuple[str, ...]: + for _ in range(200): + if recorder.call_types: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.05) + return recorder.call_types + + +STREAM_CHUNK: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":"streamed"},"finish_reason":null}]}' +STREAM_DONE: Final = '{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}' + + +@pytest.fixture +def stream_recorder(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> _SuccessRowRecorder: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, + content=f"data: {STREAM_CHUNK}\n\ndata: {STREAM_DONE}\n\ndata: [DONE]\n\n".encode(), + headers={"content-type": "text/event-stream"}, + ) + ) + recorder: Final = _SuccessRowRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + return recorder + + +async def _read_through_message_stop(stream: AsyncIterator[bytes]) -> None: + async for frame in stream: + if frame.startswith(b"event: message_stop"): + return + + +@pytest.mark.asyncio +async def test_bridged_stream_logs_one_row_when_the_client_stream_ends(stream_recorder: _SuccessRowRecorder): + stream: Final = await _ask({"stream": True}) + assert isinstance(stream, AnthropicSSEStream) + + await _read_through_message_stop(stream) + rows_while_the_client_stream_is_open: Final = await _settled_rows(stream_recorder) + remaining: Final[list[bytes]] = [frame async for frame in stream] + + assert rows_while_the_client_stream_is_open == () + assert remaining == [] + assert await _settled_rows(stream_recorder) == ("anthropic_messages",) + + +@pytest.mark.asyncio +async def test_bridged_stream_still_logs_when_the_client_leaves_after_message_stop( + stream_recorder: _SuccessRowRecorder, +): + stream: Final = await _ask({"stream": True}) + assert isinstance(stream, AnthropicSSEStream) + + await _read_through_message_stop(stream) + await stream.aclose() + + assert await _settled_rows(stream_recorder) == ("anthropic_messages",) + + +@pytest.mark.asyncio +async def test_bridged_stream_holds_its_row_for_the_proxy_end_of_stream_guardrails( + stream_recorder: _SuccessRowRecorder, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "_async_success_callback", [stream_recorder]) + logging_obj: Final = Logging( + model="hosted_vllm/served-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="6825beef-0000-4000-8000-000000000007", + function_id="1234", + ) + + async def release(*held: object) -> None: + return None + + setattr(logging_obj, "_on_deferred_stream_complete", release) # noqa: B010 # the proxy arms this slot dynamically + stream: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + max_tokens=10, + messages=[{"role": "user", "content": "hi"}], + model="hosted_vllm/served-model", + stream=True, + api_base=API_BASE, + api_key="sk-chat", + litellm_logging_obj=logging_obj, + ) + assert isinstance(stream, AnthropicSSEStream) + frames: Final = [frame async for frame in stream] + rows_before_release: Final = await _settled_rows(stream_recorder) + held: Final[object] = getattr(logging_obj, "_deferred_stream_complete_args", None) + + assert frames[-1].startswith(b"event: message_stop") + assert rows_before_release == () + (parked,) = _PARKED_COMPLETION.validate_python(held) + assert asyncio.iscoroutine(parked) + await _release(parked) + assert stream_recorder.call_types == ("anthropic_messages",) + + +def test_sync_bridged_stream_logs_one_row(stream_recorder: _SuccessRowRecorder, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "success_callback", [stream_recorder]) + logging_obj: Final = Logging( + model="hosted_vllm/served-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="6825beef-0000-4000-8000-000000000006", + function_id="1234", + ) + + stream: Final = _sync_bridge(logging_obj, stream=True) + assert isinstance(stream, Iterator) + frames: Final = list(cast(Iterator[bytes], stream)) # cast-ok: the sync bridge streams SSE frames as bytes + + assert frames[-1].startswith(b"event: message_stop") + assert asyncio.run(_settled_rows(stream_recorder)) == ("anthropic_messages",) + + +@pytest.mark.asyncio +async def test_bridged_empty_stream_still_logs_one_row(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, content=b"data: [DONE]\n\n", headers={"content-type": "text/event-stream"}) + ) + recorder: Final = _SuccessRowRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + + # deepinfra sends no stream_options, so the empty stream assembles no response to log + stream: Final = await litellm.anthropic.messages.acreate( + model="deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct", + api_base=API_BASE, + api_key="sk-chat", + max_tokens=10, + messages=[{"role": "user", "content": "hi"}], + stream=True, + ) + assert isinstance(stream, AnthropicSSEStream) + frames: Final = [frame async for frame in stream] + + assert frames[-1].startswith(b"event: message_stop") + assert await _settled_rows(recorder) == ("anthropic_messages",) diff --git a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py index 2cc0d86a3a8..95153001722 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py @@ -19,9 +19,9 @@ import json import os import sys import threading -from collections.abc import Callable, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence from datetime import datetime -from typing import Final, Iterator, List, Optional +from typing import Final, List, Optional from unittest.mock import MagicMock import httpx @@ -40,11 +40,12 @@ from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, _mid_stream_error_sse_event, ) +from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion from litellm.llms.bedrock.common_utils import BedrockError from litellm.types.utils import Delta, ModelResponseStream, StandardLoggingPayload, StreamingChoices, Usage -def _make_chunk(delta: Delta, finish_reason: Optional[str] = None) -> MagicMock: +def _make_chunk(delta: Delta, finish_reason: str | None = None) -> MagicMock: chunk = MagicMock() chunk.choices = [ StreamingChoices(finish_reason=finish_reason, index=0, delta=delta, logprobs=None) @@ -78,7 +79,7 @@ def _parse_sse(raw: bytes) -> tuple[str, dict]: return event_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: ")) -async def _drain_sse(wrapper: AnthropicStreamWrapper) -> List[bytes]: +async def _drain_sse(wrapper: AnthropicStreamWrapper) -> list[bytes]: return [event async for event in wrapper.async_anthropic_sse_wrapper()] @@ -385,3 +386,51 @@ async def test_mid_stream_error_emits_error_event_without_logging_obj(): events = await _drain_sse(_failing_wrapper(None)) assert _parse_sse(events[-1])[0] == "error" + + +class _AsyncSuccessRecorder(CustomLogger): + calls: tuple[object, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.calls = (*self.calls, response_obj) + + +class _CompletesThenRaises: + def __init__(self, exc: Exception, held: HeldStreamCompletion) -> None: + self._exc: Final = exc + self._held: Final = held + + def __iter__(self) -> Iterator[ModelResponseStream]: + return iter(()) + + def __aiter__(self) -> AsyncIterator[ModelResponseStream]: + return self._chunks() + + async def _chunks(self) -> AsyncIterator[ModelResponseStream]: + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="partial"))]) + self._held.take(litellm.ModelResponse()) + raise self._exc + + +@pytest.mark.asyncio +async def test_a_stream_that_fails_after_the_executor_completed_logs_no_success_row(): + success_recorder: Final = _AsyncSuccessRecorder() + async_recorder: Final = _AsyncFailureRecorder() + logging_obj: Final = _make_logging_obj("fails-after-complete", async_recorder, _SyncFailureRecorder()) + logging_obj.dynamic_async_success_callbacks = [success_recorder] + held: Final = HeldStreamCompletion(logging_obj) + wrapper: Final = AnthropicStreamWrapper( + completion_stream=_CompletesThenRaises(_bedrock_drop(), held), + model="bedrock-converse-sonnet-4-6", + litellm_logging_obj=logging_obj, + held_completion=held, + ) + + events: Final = await _drain_sse(wrapper) + await asyncio.sleep(0.2) + + assert _parse_sse(events[-1])[0] == "error" + assert [str(exc) for exc in async_recorder.exceptions] == ["ConverseStream ended without messageStop"] + assert success_recorder.calls == () diff --git a/tests/unit/llms/anthropic/messages/context_management/test_constants.py b/tests/unit/llms/anthropic/messages/context_management/test_constants.py index da0d9c38791..9870da8ab04 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_constants.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_constants.py @@ -103,15 +103,18 @@ async def _mock_streaming_chunks(): ) +_PLANNED_CHAT = "litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat" + + @pytest.mark.asyncio async def test_polyfill_round_trip_non_streaming(): captured = {} - async def fake_acompletion(**kwargs): - captured.update(kwargs) + async def fake_chat(plan): + captured.update({**plan.kwargs, **plan.named}) return _mock_completion_response() - with patch("litellm.acompletion", side_effect=fake_acompletion): + with patch(_PLANNED_CHAT, side_effect=fake_chat): response = await litellm.anthropic.messages.acreate( model=MODEL, messages=_make_history(n_pairs=5), @@ -173,11 +176,11 @@ async def test_polyfill_round_trip_non_streaming(): async def test_polyfill_trigger_not_met_passes_through_unchanged(): captured = {} - async def fake_acompletion(**kwargs): - captured.update(kwargs) + async def fake_chat(plan): + captured.update({**plan.kwargs, **plan.named}) return _mock_completion_response() - with patch("litellm.acompletion", side_effect=fake_acompletion): + with patch(_PLANNED_CHAT, side_effect=fake_chat): response = await litellm.anthropic.messages.acreate( model=MODEL, messages=_make_history(n_pairs=2), @@ -209,10 +212,10 @@ async def test_polyfill_trigger_not_met_passes_through_unchanged(): @pytest.mark.asyncio async def test_polyfill_streaming_attaches_to_message_delta(): - async def fake_acompletion(**kwargs): + async def fake_chat(plan): return _mock_streaming_chunks() - with patch("litellm.acompletion", side_effect=fake_acompletion): + with patch(_PLANNED_CHAT, side_effect=fake_chat): response = await litellm.anthropic.messages.acreate( model=MODEL, messages=_make_history(n_pairs=5), diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 5ce6d03f477..79e80568fba 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -2,20 +2,20 @@ import asyncio import copy import json import os +import unittest.mock import uuid from dataclasses import dataclass +from collections.abc import Mapping from typing import Any, Dict, Final, List +from unittest.mock import AsyncMock, MagicMock, patch import httpx import responses -import respx import pytest +import respx from fastapi.testclient import TestClient from pydantic import TypeAdapter, ValidationError - -from unittest.mock import AsyncMock, MagicMock, patch - import litellm from litellm.anthropic_interface import messages from litellm.integrations.custom_logger import CustomLogger @@ -30,7 +30,6 @@ from litellm.types.utils import ( StreamingChoices, ) from litellm.utils import _invalidate_model_cost_lowercase_map -import unittest.mock @pytest.mark.asyncio @@ -340,8 +339,10 @@ async def test_bedrock_converse_budget_tokens_preserved(): usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ) - with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: - mock_acompletion.return_value = mock_response + with patch( + "litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat", new_callable=AsyncMock + ) as mock_chat: + mock_chat.return_value = mock_response try: await messages.acreate( @@ -353,13 +354,9 @@ async def test_bedrock_converse_budget_tokens_preserved(): except (ValueError, TypeError, AttributeError): pass # Expected due to response format conversion - mock_acompletion.assert_called_once() + mock_chat.assert_called_once() - call_kwargs = mock_acompletion.call_args.kwargs - print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str)) - - # Verify thinking parameter is passed through with budget_tokens preserved - thinking_param = call_kwargs.get("thinking") + thinking_param = mock_chat.call_args.args[0].named.get("thinking") assert thinking_param is not None, "thinking parameter should be passed to acompletion" assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'" assert thinking_param.get("budget_tokens") == 1024, ( @@ -1241,7 +1238,10 @@ async def test_messages_streaming_reports_provider_local_model(requested_model, async def __anext__(self): raise StopAsyncIteration - with patch("litellm.acompletion", new=AsyncMock(return_value=_EmptyStream())): + with patch( + "litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat", + new=AsyncMock(return_value=_EmptyStream()), + ): stream = await litellm.anthropic.messages.acreate( max_tokens=100, messages=[{"role": "user", "content": "ping"}], @@ -2455,3 +2455,81 @@ async def test_anthropic_messages_strict_tools_rejection_is_sent_again_without_s assert first["tools"] == [tool] assert second == {**first, "tools": [{"name": "lookup_record", "input_schema": {"type": "object"}}]} assert _JSON_BODY.validate_python(response)["id"] == "msg_strict_retry" + + +_REASONING_RESPONSE: Final = { + "id": "resp_thinking", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.5", + "output": [ + {"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "worked it out"}]}, + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "the answer", "annotations": []}], + }, + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} + + +class _DeploymentHookCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None: + self.calls += 1 + + +@pytest.mark.asyncio +async def test_forced_chat_sends_openai_thinking_to_responses_through_the_chat_bridge( + monkeypatch: pytest.MonkeyPatch, +): + counter: Final = _DeploymentHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=False) as upstream: + responses_route: Final = upstream.post("http://openai.test/v1/responses").mock( + return_value=httpx.Response(200, json=_REASONING_RESPONSE) + ) + chat_route: Final = upstream.post("http://openai.test/v1/chat/completions") + response: Final = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.5", + api_base="http://openai.test/v1", + api_key="sk-test", + max_tokens=1024, + messages=[ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "let me check"}, + {"type": "tool_use", "id": "toolu_1", "name": "get_w", "input": {"c": "a"}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny"}]}, + ], + tools=[{"name": "get_w", "description": "d", "input_schema": {"type": "object", "properties": {}}}], + thinking={"type": "enabled", "budget_tokens": 1024}, + ) + + assert isinstance(response, dict) + assert [block["type"] for block in response["content"]] == ["thinking", "text"] + assert response["content"][1]["text"] == "the answer" + assert (responses_route.call_count, chat_route.call_count) == (1, 0) + assert counter.calls == 1 + sent_input: Final = json.loads(responses_route.calls[0].request.content)["input"] + assert [item["type"] for item in sent_input] == ["message", "message", "function_call", "function_call_output"] + assert sent_input[1]["role"] == "assistant" + assert sent_input[3]["output"] == [{"type": "input_text", "text": "sunny"}] diff --git a/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py index 7334b4eeaa1..00a3c639cbf 100644 --- a/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py @@ -32,13 +32,7 @@ def _call_handler_and_capture_optional_params(thinking=None, **extra_kwargs): with patch( "litellm.llms.anthropic.messages.native.handler." "base_llm_http_handler" - ) as mock_handler, patch( - "litellm.llms.anthropic.messages.native.handler." - "ProviderConfigManager" - ) as mock_pcm: - # Make get_provider_anthropic_messages_config return a non-None config - # so the handler takes the native Anthropic path - mock_pcm.get_provider_anthropic_messages_config.return_value = MagicMock() + ) as mock_handler: mock_handler.anthropic_messages_handler.return_value = MagicMock() kwargs = dict(extra_kwargs) diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 382917fc756..259ac0efd2d 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -1,30 +1,40 @@ +import asyncio import datetime import json import os import sys -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Mapping from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch +import httpx import pytest import respx +from pydantic import TypeAdapter sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_prompt_management import CustomPromptManagement +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, ) from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, - _build_responses_kwargs, + _build_responses_request, ) +from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import AnthropicResponsesSSEByteStream from litellm.router import Router -from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested +from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS +from litellm.types.llms.openai import AllMessageValues +from litellm.types.prompts.init_prompts import PromptSpec +from litellm.types.utils import StandardCallbackDynamicParams -MESSAGES = [{"role": "user", "content": "hello"}] +MESSAGES: Final[list[dict[str, object]]] = [{"role": "user", "content": "hello"}] CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) RESPONSES_SSE_BODY = ( @@ -40,20 +50,8 @@ RESPONSES_SSE_BODY = ( ) -def test_build_responses_kwargs_never_asks_the_nested_call_to_replay_a_stream(): - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=1024, - messages=MESSAGES, - model="azure/gpt-5.6-luna", - extra_kwargs={"custom_llm_provider": "azure", **{key: True for key in CONVERTED_STREAM_KEYS}}, - ) - - assert not converted_stream_requested(responses_kwargs) - assert responses_kwargs["custom_llm_provider"] == "azure" - - -def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_derives_prompt_cache_key_from_claude_code_session_id(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", @@ -64,8 +62,8 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_sessio assert responses_kwargs["prompt_cache_key"] == "session-abc" -def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_sets_no_prompt_cache_key_for_plain_user_id(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", @@ -76,8 +74,8 @@ def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id(): assert "prompt_cache_key" not in responses_kwargs -def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_prefers_explicit_prompt_cache_key_over_derived(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", @@ -88,8 +86,8 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived() assert responses_kwargs["prompt_cache_key"] == "explicit-key" -def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_asks_openai_for_encrypted_reasoning_without_thinking(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", @@ -99,8 +97,8 @@ def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thin assert "reasoning" not in responses_kwargs -def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_skips_include_for_a_responses_provider_that_rejects_it(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="perplexity/sonar", @@ -118,11 +116,11 @@ def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_reje ("openai/gpt-5.6-luna", "openai", "detailed"), ], ) -def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_accepts( +def test_build_responses_request_auto_summary_asks_each_target_for_a_level_it_accepts( monkeypatch: pytest.MonkeyPatch, model: str, provider: str, summary: str ) -> None: monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( max_tokens=1024, messages=MESSAGES, model=model, @@ -132,8 +130,8 @@ def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_acc assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary -def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_keeps_the_deployment_include_next_to_encrypted_reasoning(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", @@ -142,8 +140,8 @@ def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_r assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"] -def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): - responses_kwargs = _build_responses_kwargs( +def test_build_responses_request_without_metadata_sets_no_prompt_cache_key(): + responses_kwargs = _build_responses_request( max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", @@ -194,8 +192,6 @@ async def test_streaming_hands_the_logging_object_the_message_id_the_caller_is_s The bridge mints the ``msg_`` id itself, and it is the only request id a streaming /v1/messages caller ever sees, so the spend row has to be keyed on that same value. """ - from litellm.litellm_core_utils.litellm_logging import Logging - monkeypatch.setenv("OPENAI_API_KEY", "sk-lit6825-test") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() @@ -425,7 +421,7 @@ def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> di def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_the_same_block( extra_kwargs: dict[str, object], ) -> None: - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( model="openai/gpt-5.6", extra_kwargs=extra_kwargs, **_request_with_client_cache_marks({**EPHEMERAL, "ttl": "1h"}), @@ -453,7 +449,7 @@ def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_th def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5_6_on_that_screenshot() -> None: - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, @@ -532,7 +528,7 @@ RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoi def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drops_them_for_a_model_that_takes_none( model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object] ) -> None: - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( model=model, extra_kwargs=extra_kwargs, max_tokens=1024, @@ -558,7 +554,7 @@ def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drop def _gpt_5_6_input_for_a_cached_block(drop_params: bool, raw_mark: Mapping[str, object]) -> object: - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( model="openai/gpt-5.6", extra_kwargs={**OPENAI_API, "drop_params": drop_params}, max_tokens=1024, @@ -589,10 +585,10 @@ def test_a_malformed_raw_marker_beside_cache_control_gives_way_to_it_only_under_ def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_read_prompt_cache_breakpoint( model: str, extra_kwargs: dict[str, object] ) -> None: - marked: Final = _build_responses_kwargs( + marked: Final = _build_responses_request( model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(EPHEMERAL) ) - unmarked: Final = _build_responses_kwargs( + unmarked: Final = _build_responses_request( model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(None) ) @@ -623,7 +619,7 @@ def _tool_result_turn_with(text_block: Mapping[str, object]) -> list[dict[str, o @pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) def test_malformed_tool_result_text_answers_400_naming_the_block(text_block: Mapping[str, object]) -> None: with pytest.raises(litellm.BadRequestError) as rejected: - _build_responses_kwargs( + _build_responses_request( model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, @@ -650,7 +646,7 @@ def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None: ] with pytest.raises(litellm.BadRequestError) as rejected: - _build_responses_kwargs(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages) + _build_responses_request(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages) assert rejected.value.status_code == 400 assert f"tool_result {tool_use_id} content.0.text: Input should be a valid string" in rejected.value.message @@ -659,7 +655,7 @@ def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None: @pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, object]) -> None: - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( model="openai/gpt-5.6", extra_kwargs={**OPENAI_API, "drop_params": True}, max_tokens=1024, @@ -672,7 +668,7 @@ def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, def test_tool_result_text_block_without_text_still_reaches_the_model_as_empty_text() -> None: - responses_kwargs: Final = _build_responses_kwargs( + responses_kwargs: Final = _build_responses_request( model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, @@ -1084,3 +1080,344 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( ] message_delta: Final = next(event for event in events if event["type"] == "message_delta") assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 +SEARCH_MODEL: Final = "gpt-bridge-search-cost" +SEARCH_RATES: Final = {"input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05} +SEARCH_COST_PER_QUERY: Final = 0.03 +SEARCH_RESPONSE: Final = { + **REASONING_RESPONSE, + "id": "resp_bridge_search", + "model": SEARCH_MODEL, + "output": [ + *( + { + "type": "web_search_call", + "id": f"ws_{i}", + "status": "completed", + "action": {"type": "search", "query": "q"}, + } + for i in range(3) + ), + REASONING_RESPONSE["output"][1], + ], + "usage": {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120}, +} + + +@pytest.mark.asyncio +async def test_served_bridge_response_is_billed_for_each_web_search_call_the_provider_made( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setenv("OPENAI_API_KEY", "sk-bridge-search-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setitem( + litellm.model_cost, + SEARCH_MODEL, + { + "litellm_provider": "openai", + "mode": "responses", + "search_context_cost_per_query": { + f"search_context_size_{size}": SEARCH_COST_PER_QUERY for size in ("low", "medium", "high") + }, + **SEARCH_RATES, + }, + ) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").respond(status_code=200, json=SEARCH_RESPONSE) + logging_obj: Final = Logging( + model=SEARCH_MODEL, + messages=MESSAGES, + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="bridge-search-0000-4000-8000-000000000001", + function_id="1234", + ) + + served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=412, + messages=MESSAGES, + model=f"openai/{SEARCH_MODEL}", + custom_llm_provider="openai", + tools=[{"type": "web_search_20250305", "name": "web_search"}], + litellm_logging_obj=logging_obj, + ) + assert isinstance(served, dict) + + cost: Final = ProxyBaseLLMRequestProcessing._response_cost_from_logging_obj( + response=dict(served), logging_obj=logging_obj + ) + + expected_tokens: Final = 100 * SEARCH_RATES["input_cost_per_token"] + 20 * SEARCH_RATES["output_cost_per_token"] + assert cost == pytest.approx(expected_tokens + 3 * SEARCH_COST_PER_QUERY) + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["tool_usage_cost"] == pytest.approx(3 * SEARCH_COST_PER_QUERY) + + +def _messages_logging_obj(stream: bool) -> Logging: + return Logging( + model="gpt-5.6-luna", + messages=MESSAGES, + stream=stream, + call_type="anthropic_messages", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="6825beef-0000-4000-8000-000000000004", + function_id="1234", + ) + + +def _untyped_bridge() -> Callable[..., Coroutine[object, object, object]]: + return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler + + +async def _logged_bridge_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, upstream_response: httpx.Response, **extra: object +) -> tuple[object, respx.Route]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=upstream_response) + response: Final = await _untyped_bridge()( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + litellm_logging_obj=_messages_logging_obj(stream=False), + **extra, + ) + return response, upstream + + +@pytest.mark.asyncio +async def test_bridge_answers_a_non_stream_request_with_a_message_even_when_the_caller_asked_for_stream_replay( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + response, upstream = await _logged_bridge_call( + respx_mock, + monkeypatch, + httpx.Response(200, json=TOOL_CALL_RESPONSE), + **{key: True for key in CONVERTED_STREAM_KEYS}, + ) + + assert isinstance(response, dict) and response["type"] == "message" + assert json.loads(upstream.calls.last.request.content).get("stream") is not True + + +@pytest.mark.asyncio +async def test_bridge_sends_the_deployment_prompt_cache_options_upstream( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _, upstream = await _logged_bridge_call( + respx_mock, + monkeypatch, + httpx.Response(200, json=TOOL_CALL_RESPONSE), + prompt_cache_options={"mode": "explicit"}, + ) + + assert json.loads(upstream.calls.last.request.content)["prompt_cache_options"] == {"mode": "explicit"} + + +@pytest.mark.asyncio +async def test_bridge_raises_the_litellm_rate_limit_error_for_an_upstream_429( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.RateLimitError): + await _logged_bridge_call( + respx_mock, + monkeypatch, + httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit_error"}}), + ) + + +class _TemplatePromptManager(CustomPromptManagement): + async def async_get_chat_completion_prompt( + self, + model: str, + messages: list[AllMessageValues], + non_default_params: dict[str, object], + prompt_id: str | None, + prompt_variables: dict[str, object] | None, + dynamic_callback_params: StandardCallbackDynamicParams, + litellm_logging_obj: Logging, + prompt_spec: PromptSpec | None = None, + tools: list[dict[str, object]] | None = None, + prompt_label: str | None = None, + prompt_version: int | None = None, + ignore_prompt_manager_model: bool | None = False, + ignore_prompt_manager_optional_params: bool | None = False, + ) -> tuple[str, list[AllMessageValues], dict[str, object]]: + return model, [{"role": "user", "content": "from the template"}], non_default_params + + +@pytest.mark.asyncio +async def test_bridge_applies_the_prompt_template_named_by_prompt_id( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "callbacks", [_TemplatePromptManager()]) + + _, upstream = await _logged_bridge_call( + respx_mock, monkeypatch, httpx.Response(200, json=TOOL_CALL_RESPONSE), prompt_id="bridge_template" + ) + + assert "from the template" in upstream.calls.last.request.content.decode() + + +def _logged_sync_bridge_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, upstream_response: httpx.Response +) -> tuple[object, respx.Route]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=upstream_response) + response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + litellm_logging_obj=_messages_logging_obj(stream=False), + ) + return response, upstream + + +def test_sync_bridge_answers_with_a_message_from_one_upstream_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + response, upstream = _logged_sync_bridge_call(respx_mock, monkeypatch, httpx.Response(200, json=TOOL_CALL_RESPONSE)) + + assert isinstance(response, dict) and response["type"] == "message" + assert [block["id"] for block in response["content"] if block["type"] == "tool_use"] == ["call_curl"] + assert upstream.call_count == 1 + + +def test_sync_bridge_raises_the_litellm_rate_limit_error_for_an_upstream_429( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + with pytest.raises(litellm.RateLimitError): + _logged_sync_bridge_call( + respx_mock, + monkeypatch, + httpx.Response(429, json={"error": {"message": "slow down", "type": "rate_limit_error"}}), + ) + + +class _SuccessRowRecorder(CustomLogger): + call_types: tuple[str, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.call_types = (*self.call_types, str(kwargs.get("call_type"))) + + +_PARKED_COMPLETION: Final = TypeAdapter(tuple[object]) + + +async def _release(parked: Coroutine[object, object, object]) -> None: + await parked + + +async def _settled_rows(recorder: _SuccessRowRecorder) -> tuple[str, ...]: + for _ in range(200): + if recorder.call_types: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.05) + return recorder.call_types + + +async def _bridged_stream( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, logging_obj: Logging +) -> tuple[AnthropicResponsesSSEByteStream, _SuccessRowRecorder]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-native-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + recorder: Final = _SuccessRowRecorder() + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + ) + stream: Final = await _untyped_bridge()( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + stream=True, + custom_llm_provider="openai", + litellm_logging_obj=logging_obj, + ) + assert isinstance(stream, AnthropicResponsesSSEByteStream) + return stream, recorder + + +@pytest.mark.asyncio +async def test_bridged_stream_opens_with_the_cost_breakdown_headers( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import cost_headers + + logging_obj: Final = _messages_logging_obj(stream=True) + stream, _ = await _bridged_stream(respx_mock, monkeypatch, logging_obj) + + headers: Final = cost_headers(None, logging_obj) + await stream.aclose() + + assert headers["x-litellm-response-cost-output"] == "0.0" + assert headers["x-litellm-response-cost-discount-amount"] == "0.0" + assert headers["x-litellm-response-cost-margin-amount"] == "0.0" + + +async def _read_through_message_stop(stream: AsyncIterator[bytes]) -> None: + async for frame in stream: + if frame.startswith(b"event: message_stop"): + return + + +@pytest.mark.asyncio +async def test_bridged_stream_logs_one_row_when_the_client_stream_ends( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + stream, recorder = await _bridged_stream(respx_mock, monkeypatch, _messages_logging_obj(stream=True)) + + await _read_through_message_stop(stream) + rows_while_the_client_stream_is_open: Final = await _settled_rows(recorder) + remaining: Final[list[bytes]] = [frame async for frame in stream] + + assert rows_while_the_client_stream_is_open == () + assert remaining == [] + assert await _settled_rows(recorder) == ("anthropic_messages",) + + +@pytest.mark.asyncio +async def test_bridged_stream_still_logs_when_the_client_leaves_after_message_stop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + stream, recorder = await _bridged_stream(respx_mock, monkeypatch, _messages_logging_obj(stream=True)) + + await _read_through_message_stop(stream) + await stream.aclose() + + assert await _settled_rows(recorder) == ("anthropic_messages",) + + +@pytest.mark.asyncio +async def test_bridged_stream_holds_its_row_for_the_proxy_end_of_stream_guardrails( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + logging_obj: Final = _messages_logging_obj(stream=True) + + async def release(*held: object) -> None: + return None + + setattr(logging_obj, "_on_deferred_stream_complete", release) # noqa: B010 # the proxy arms this slot dynamically + stream, recorder = await _bridged_stream(respx_mock, monkeypatch, logging_obj) + + frames: Final = [frame async for frame in stream] + rows_before_release: Final = await _settled_rows(recorder) + held: Final[object] = getattr(logging_obj, "_deferred_stream_complete_args", None) + + assert frames[-1].startswith(b"event: message_stop") + assert rows_before_release == () + (parked,) = _PARKED_COMPLETION.validate_python(held) + assert asyncio.iscoroutine(parked) + await _release(parked) + assert recorder.call_types == ("anthropic_messages",) diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index 4626f76c8fc..922458b2fb0 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -8,7 +8,7 @@ import datetime import json import os import sys -from collections.abc import AsyncIterator, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from types import SimpleNamespace import anyio @@ -18,10 +18,12 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../. import litellm from litellm.exceptions import MidStreamFallbackError +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) +from litellm.llms.anthropic.messages.held_stream_completion import HeldStreamCompletion from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt from litellm.llms.anthropic.messages.native.utils import INCOMPLETE_STREAM_ERROR_MESSAGE from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( @@ -818,3 +820,56 @@ class TestUpstreamFailureReachesTheProxyFailureHook: asyncio.run(_disconnect_mid_hook()) assert len(finished) == 1 + + +class _AsyncSuccessRecorder(CustomLogger): + calls: tuple[object, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.calls = (*self.calls, response_obj) + + +def test_a_stream_that_fails_after_the_executor_completed_logs_no_success_row(): + recorder = _AsyncSuccessRecorder() + logging_obj = LitellmLogging( + model="m", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="fails-after-complete", + function_id="fails-after-complete", + dynamic_async_success_callbacks=[recorder], + ) + held = HeldStreamCompletion(logging_obj) + completed = ResponsesAPIResponse( + id="resp_1", + created_at=1, + status="completed", + model="m", + output=[], + parallel_tool_calls=True, + tool_choice="auto", + tools=[], + ) + + async def _gen() -> AsyncIterator[dict[str, object]]: + yield {"type": "response.created"} + held.take(completed) + yield {"type": "response.completed", "response": completed} + raise ConnectionResetError("Response payload is not completed") + + async def _run() -> list[bytes]: + wrapper = AnthropicResponsesStreamWrapper( + responses_stream=_gen(), model="m", litellm_logging_obj=logging_obj, held_completion=held + ) + frames: list[bytes] = [frame async for frame in wrapper.async_anthropic_sse_wrapper()] + await asyncio.sleep(0.2) + return frames + + frames = asyncio.run(_run()) + + assert frames[-1].startswith(b"event: error") + assert recorder.calls == () diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index f6fcb9e6f1e..d1f05f84bab 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -2411,16 +2411,3 @@ class TestPromptCacheBreakpointToResponses: self.MARKED_SCREENSHOT_PART, ] assert log_output == self.LOG_OUTPUT - - def test_prompt_cache_options_forwarded_to_responses_kwargs(self): - from litellm.llms.anthropic.messages.responses_adapters.handler import ( - _build_responses_kwargs, - ) - - kwargs = _build_responses_kwargs( - max_tokens=16, - messages=[{"role": "user", "content": "hi"}], - model="openai/gpt-5.6", - extra_kwargs={"prompt_cache_options": {"mode": "explicit"}}, - ) - assert kwargs["prompt_cache_options"] == {"mode": "explicit"} diff --git a/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py index 1cdc288ff87..79d87ee24ad 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py @@ -172,7 +172,7 @@ def test_azure_o_series_routing(): assert "stream" not in mock_create.call_args.kwargs["body"] -@patch("litellm.main.azure_o1_chat_completions._get_openai_client") +@patch("litellm.native.chat_providers.handlers.azure_o1_chat_completions._get_openai_client") def test_openai_o_series_max_retries_0(mock_get_openai_client): import litellm diff --git a/tests/unit/llms/azure/test_azure.py b/tests/unit/llms/azure/test_azure.py index c52321baf02..c331890937d 100644 --- a/tests/unit/llms/azure/test_azure.py +++ b/tests/unit/llms/azure/test_azure.py @@ -583,7 +583,9 @@ def test_azure_openai_responses_bridge(): litellm.turn_on_debug() - with patch.object(litellm, "responses") as mock_responses: + from litellm.responses import bridge_executor + + with patch.object(bridge_executor, "native_responses") as mock_responses: try: response = completion( model="azure/responses/test-azure-computer-use-preview", @@ -597,10 +599,10 @@ def test_azure_openai_responses_bridge(): mock_responses.assert_called_once() assert ( - mock_responses.call_args.kwargs["model"] + mock_responses.call_args.args[0]["model"] == "azure/test-azure-computer-use-preview" ) - assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure" + assert mock_responses.call_args.args[0]["custom_llm_provider"] == "azure" def test_azure_with_content_safety_error(): diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 8f083298555..47c23a52cb5 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -735,7 +735,7 @@ async def test_ensure_initialize_azure_sdk_client_always_used_azure_text(call_ty # Get appropriate input for this call type input_kwarg = test_inputs.get(call_type.value, {}) - patch_target = "litellm.main.azure_text_completions.initialize_azure_sdk_client" + patch_target = "litellm.native.chat_providers.handlers.azure_text_completions.initialize_azure_sdk_client" # Mock the initialize_azure_sdk_client function with patch(patch_target) as mock_init_azure: diff --git a/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py b/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py index e08098e9aab..9a48aafdea6 100644 --- a/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py +++ b/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py @@ -21,7 +21,7 @@ def test_main_azure_ai_claude_completion_passes_timeout_to_azure_anthropic_handl captured.update(kwargs) return ModelResponse() - with patch("litellm.main.azure_anthropic_chat_completions") as mock_azure_anthropic: + with patch("litellm.native.chat_providers.azure.azure_anthropic_chat_completions") as mock_azure_anthropic: mock_azure_anthropic.completion = MagicMock( side_effect=fake_azure_anthropic_completion ) diff --git a/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py b/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py index 3d15a2e8870..440a5455209 100644 --- a/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py +++ b/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py @@ -686,7 +686,7 @@ class TestGDCGeminiConfig: class TestCompleteGDC: @patch("litellm.main.base_llm_http_handler.completion") def test_complete_gdc_resolves_key_and_base(self, mock_completion, monkeypatch): - from litellm.main import gdc_transformation + from litellm.native.chat_providers.handlers import gdc_transformation mock_completion.return_value = MagicMock() monkeypatch.setattr(litellm, "gdc_key", "resolved-key", raising=False) diff --git a/tests/unit/llms/gemini/test_gemini_client_setup.py b/tests/unit/llms/gemini/test_gemini_client_setup.py index 48b010aca48..2aac6d69163 100644 --- a/tests/unit/llms/gemini/test_gemini_client_setup.py +++ b/tests/unit/llms/gemini/test_gemini_client_setup.py @@ -28,7 +28,7 @@ def test_gemini_completion_no_api_key(): del os.environ[key] # Test without mock_response to ensure actual API key validation - with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: + with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], @@ -60,7 +60,7 @@ def test_gemini_completion_no_api_key_with_mock(): with patch("litellm.get_secret") as mock_get_secret: mock_get_secret.return_value = None - with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: + with pytest.raises(Exception, match='in complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], diff --git a/tests/unit/llms/github_copilot/test_per_user_auth.py b/tests/unit/llms/github_copilot/test_per_user_auth.py index cf6fb9fa71f..94fd218f7aa 100644 --- a/tests/unit/llms/github_copilot/test_per_user_auth.py +++ b/tests/unit/llms/github_copilot/test_per_user_auth.py @@ -668,7 +668,7 @@ def test_e2e_completion_per_user_credential(per_user_credential): def test_e2e_completion_per_user_session_token_wins_over_caller_authorization(per_user_credential): """The session token reaches the wire even when the caller passes their own - Authorization in extra_headers (chat dispatches through _complete_custom_openai).""" + Authorization in extra_headers (chat dispatches through complete_custom_openai).""" with ( github_http(_exchange_responder()), _no_authenticator(), diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 5b27a446fb3..1058ac5fd36 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -1934,7 +1934,9 @@ def test_responses_gpt54_with_xhigh_reasoning(): Ensure chat->responses bridge sends the correct request payload for openai/responses/gpt-5.4 with reasoning_effort="xhigh". """ - with patch("litellm.responses") as mock_responses: + from litellm.responses import bridge_executor + + with patch.object(bridge_executor, "native_responses") as mock_responses: mock_responses.side_effect = RuntimeError("stop_after_request_build") with pytest.raises(litellm.APIConnectionError): @@ -1946,7 +1948,7 @@ def test_responses_gpt54_with_xhigh_reasoning(): ) mock_responses.assert_called_once() - request_body = mock_responses.call_args.kwargs + request_body = mock_responses.call_args.args[0] assert request_body["model"] == "openai/gpt-5.4" diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 67c78b62151..e92218dacd8 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -8515,8 +8515,11 @@ async def test_claude_tool_use_with_gemini(): return chunk raise StopAsyncIteration - with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: - mock_acompletion.return_value = MockAsyncStream() + with patch( + "litellm.llms.anthropic.messages.adapters.handler.aexecute_planned_chat", + new_callable=AsyncMock, + ) as mock_chat: + mock_chat.return_value = MockAsyncStream() response = await litellm.anthropic.messages.acreate( messages=[ diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 7e0baaf9e10..be745e318f9 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -1545,9 +1545,9 @@ def test_system_prompt_only_adds_blank_user_message(): @pytest.fixture def _cached_vertex_credentials(monkeypatch: pytest.MonkeyPatch) -> str: - from litellm.main import ( + from litellm.main import vertex_embedding + from litellm.native.chat_providers.handlers import ( vertex_chat_completion, - vertex_embedding, vertex_model_garden_chat_completion, vertex_partner_models_chat_completion, ) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py index 98071594ebf..7dde7e6b38d 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py @@ -15,6 +15,7 @@ import pytest import litellm from litellm._uuid import uuid +from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, convert_to_anthropic_tool_result, @@ -406,7 +407,7 @@ async def test_vertex_ai_anthropic_async_completion_inlines_https_images_off_the with ( patch.dict(sys.modules, {"vertexai": vertexai}), patch.object( # test-quality-ok: litellm.acompletion has no seam for Vertex token minting - litellm.main.vertex_partner_models_chat_completion, + vertex_partner_models_chat_completion, "_ensure_access_token", return_value=("token", "test-project"), ), diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 3ae208bb3a8..c8c4a80a5d4 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -585,7 +585,7 @@ async def test_vertex_ai_anthropic_prompt_caching_does_not_add_beta_header( monkeypatch: pytest.MonkeyPatch, sync_mode: bool, ) -> None: - from litellm.main import vertex_partner_models_chat_completion + from litellm.native.chat_providers.handlers import vertex_partner_models_chat_completion monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setitem( diff --git a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py index 0726ebb5ee1..16de483176f 100644 --- a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py @@ -1320,7 +1320,7 @@ def _gemma_cached_access_token(): """Serve a fake token from the handler's credential cache so no auth round-trip runs.""" from types import SimpleNamespace - from litellm.main import vertex_gemma_chat_completion + from litellm.native.chat_providers.handlers import vertex_gemma_chat_completion cache = vertex_gemma_chat_completion._credentials_project_mapping key = (_FAKE_GEMMA_CREDENTIALS, "test") diff --git a/tests/unit/native/__init__.py b/tests/unit/native/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/native/test_chat_plan.py b/tests/unit/native/test_chat_plan.py new file mode 100644 index 00000000000..f8ef0cb5a65 --- /dev/null +++ b/tests/unit/native/test_chat_plan.py @@ -0,0 +1,166 @@ +import json +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.exceptions import JSONSchemaValidationError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.native.chat_plan import plan_chat + +API_BASE: Final = "https://bridge-parity.invalid/v1" +SCHEMA: Final = { + "type": "object", + "properties": {"b": {"type": "integer"}}, + "required": ["b"], + "additionalProperties": False, +} +OFF_SCHEMA_TEXT: Final = '{"b": 2, "extra": true}' +_CHAT_REPLY: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "m", + "choices": [{"index": 0, "message": {"role": "assistant", "content": OFF_SCHEMA_TEXT}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, +} + + +def _router(litellm_params: dict[str, object], model_info: dict[str, object] | None = None) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "m", + "litellm_params": {"api_base": API_BASE, "api_key": "k", **litellm_params}, + **({"model_info": model_info} if model_info is not None else {}), + } + ], + num_retries=0, + ) + + +@pytest.fixture(autouse=True) +def _offline(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.parametrize( + ("litellm_params", "model_info"), + [ + ({"model": "fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct"}, None), + ({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, None), + ], +) +@pytest.mark.asyncio +async def test_a_messages_request_bridged_to_chat_validates_its_json_schema( + litellm_params: dict[str, object], + model_info: dict[str, object] | None, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "enable_json_schema_validation", True) + respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=_CHAT_REPLY)) + + with pytest.raises(JSONSchemaValidationError): + await _router(litellm_params, model_info).aanthropic_messages( + model="m", + max_tokens=32, + messages=[{"role": "user", "content": "hi"}], + output_format={"type": "json_schema", "schema": SCHEMA}, + ) + + +@pytest.mark.asyncio +async def test_a_responses_request_bridged_to_chat_validates_its_json_schema( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "enable_json_schema_validation", True) + respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, json=_CHAT_REPLY)) + + with pytest.raises(JSONSchemaValidationError): + await _router({"model": "deepseek/deepseek-reasoner"}, {"mode": "responses"}).aresponses( + model="m", + input="hi", + text={"format": {"type": "json_schema", "name": "x", "strict": True, "schema": SCHEMA}}, + ) + + +@pytest.mark.asyncio +async def test_a_provider_failure_on_the_messages_to_chat_bridge_is_mapped(respx_mock: respx.MockRouter): + respx_mock.post(f"{API_BASE}/chat/completions").mock(return_value=httpx.Response(200, content=b"not json")) + + with pytest.raises(litellm.exceptions.APIConnectionError): + await _router({"model": "groq/llama-3.1-8b-instant"}, {"mode": "responses"}).aanthropic_messages( + model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}] + ) + + +@pytest.mark.asyncio +async def test_a_responses_request_on_a_chat_api_deployment_stays_on_chat_despite_mode_responses( + respx_mock: respx.MockRouter, +): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + + await _router({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}).aresponses( + model="m", input="hi" + ) + + assert chat.call_count == 1 + + +@pytest.mark.asyncio +async def test_the_skip_bridge_flag_is_never_sent_to_the_provider(respx_mock: respx.MockRouter): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + + await _router({"model": "openai/gpt-4.1"}).acompletion( + model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True + ) + + assert "_skip_responses_api_bridge" not in json.loads(chat.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_the_skip_bridge_flag_keeps_a_mode_responses_deployment_on_chat(respx_mock: respx.MockRouter): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + + await _router({"model": "openai/gpt-4.1"}, {"mode": "responses"}).acompletion( + model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True + ) + + assert chat.call_count == 1 + + +def _bridged_chat_logging() -> Logging: + return Logging( + model="gpt-4.1", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="aresponses", + start_time=datetime.now(), + litellm_call_id="planned-chat-cache", + function_id="planned-chat-cache", + ) + + +@pytest.mark.parametrize(("response_cache", "planned"), [(None, True), (litellm.Cache(type="local"), False)]) +def test_a_bridged_chat_call_goes_through_acompletion_when_the_response_cache_is_on( + response_cache: litellm.Cache | None, planned: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cache", response_cache) + + plan: Final = plan_chat( + {"model": "openai/gpt-4.1", "messages": [{"role": "user", "content": "hi"}], "api_key": "k"}, + _bridged_chat_logging(), + ) + + assert (plan is not None) is planned diff --git a/tests/unit/native/test_route_plan.py b/tests/unit/native/test_route_plan.py new file mode 100644 index 00000000000..fd93de37275 --- /dev/null +++ b/tests/unit/native/test_route_plan.py @@ -0,0 +1,288 @@ +import asyncio +import json +from collections.abc import AsyncIterable, Callable, Coroutine, Mapping, Sequence +from typing import Final, cast + +import httpx +import pytest +import respx +from pydantic import TypeAdapter + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig +from litellm.messages.dispatch import anthropic_messages +from litellm.native.route_plan import ( + NativeChat, + NativeMessages, + NativeResponses, + RoutePlan, + plan_chat_route, + plan_messages_route, + plan_responses_route, +) + +MESSAGES_PASSTHROUGH: dict[str, object] = {"supported_endpoints": ["/v1/messages"]} + + +def _plan( + model: str, + provider: str, + *, + model_info: object = None, + compaction_requested: bool = False, + force_chat_completions: bool = False, +) -> RoutePlan: + return plan_messages_route( + requested_model=f"{provider}/{model}", + resolved_model=model, + custom_llm_provider=provider, + model_info=model_info, + api_base=None, + compaction_requested=compaction_requested, + force_chat_completions=force_chat_completions, + ) + + +def test_a_provider_that_speaks_messages_gets_its_own_messages_config(): + plan = _plan("claude-sonnet-4-5", "anthropic") + + assert isinstance(plan, NativeMessages) + assert isinstance(plan.config, AnthropicMessagesConfig) + + +@pytest.mark.parametrize( + ("model", "provider", "kwargs", "expected"), + [ + ("gpt-5.5", "openai", {}, NativeResponses()), + ("gpt-5.5", "openai", {"force_chat_completions": True}, NativeChat()), + ("gpt-5.5", "openai", {"compaction_requested": True}, NativeChat()), + ("my-model", "hosted_vllm", {}, NativeChat()), + ], +) +def test_messages_without_a_native_config_bridge_to_responses_only_for_openai( + model: str, provider: str, kwargs: dict[str, bool], expected: RoutePlan +): + assert _plan(model, provider, **kwargs) == expected + + +_MODEL_COST: Final = TypeAdapter(dict[str, object]) + + +def _isolate_model_cost(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_cost", _MODEL_COST.validate_python(litellm.model_cost)) + + +@pytest.mark.parametrize("cache_control_ttl", [True, False]) +def test_a_deployment_that_opts_into_messages_passthrough_skips_translation(cache_control_ttl: bool): + plan = _plan("my-model", "hosted_vllm", model_info={**MESSAGES_PASSTHROUGH, "cache_control_ttl": cache_control_ttl}) + + assert isinstance(plan, NativeMessages) + assert isinstance(plan.config, OpenAILikeAnthropicMessagesConfig) + assert plan.config.supports_cache_control_ttl() is cache_control_ttl + + +@pytest.mark.parametrize( + ("has_config", "use_chat_completions_api", "expected"), + [ + (True, False, NativeResponses()), + (True, True, NativeChat()), + (False, False, NativeChat()), + (False, True, NativeChat()), + ], +) +def test_responses_reach_chat_without_a_provider_config_or_when_asked( + has_config: bool, use_chat_completions_api: bool, expected: RoutePlan +): + config = OpenAIResponsesAPIConfig() if has_config else None + + assert plan_responses_route(config, use_chat_completions_api) == expected + + +@pytest.mark.parametrize( + ("has_config", "expected"), + [ + (True, NativeResponses()), + (False, NativeChat()), + ], +) +def test_chat_bridges_to_responses_only_when_the_provider_serves_it(has_config: bool, expected: RoutePlan): + assert plan_chat_route(OpenAIResponsesAPIConfig() if has_config else None) == expected + + +CHAT_BASE: Final = "http://chat-upstream.test/v1" +RESPONSES_BASE: Final = "http://responses-upstream.test/v1" +CHAT_MODEL: Final = "hosted_vllm/served-model" +RESPONSES_ONLY_MODEL: Final = "openai/bridge-once-model" +USER_TURN: Final = ({"role": "user", "content": "hi"},) + +_CHAT_REPLY: Final[Mapping[str, object]] = { + "id": "chatcmpl-once", + "object": "chat.completion", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "once"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, +} +_CHAT_CHUNKS: Final[tuple[Mapping[str, object], ...]] = ( + { + "id": "chatcmpl-once", + "object": "chat.completion.chunk", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "once"}, "finish_reason": None}], + }, + { + "id": "chatcmpl-once", + "object": "chat.completion.chunk", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "delta": dict[str, object](), "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, +) +_OUTPUT_MESSAGE: Final[Mapping[str, object]] = { + "type": "message", + "id": "msg_once", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "once", "annotations": list[object]()}], +} +_RESPONSES_REPLY: Final[Mapping[str, object]] = { + "id": "resp_once", + "object": "response", + "created_at": 1, + "model": "bridge-once-model", + "status": "completed", + "output": [_OUTPUT_MESSAGE], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": list[object](), + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, +} +_RESPONSES_EVENTS: Final[tuple[Mapping[str, object], ...]] = ( + {"type": "response.created", "sequence_number": 0, "response": {**_RESPONSES_REPLY, "status": "in_progress"}}, + {"type": "response.output_item.added", "sequence_number": 1, "output_index": 0, "item": _OUTPUT_MESSAGE}, + { + "type": "response.output_text.delta", + "sequence_number": 2, + "item_id": "msg_once", + "output_index": 0, + "content_index": 0, + "delta": "once", + }, + {"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _OUTPUT_MESSAGE}, + {"type": "response.completed", "sequence_number": 4, "response": _RESPONSES_REPLY}, +) + + +def _sse(events: Sequence[Mapping[str, object]], *, done: bool) -> bytes: + frames: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + return (frames + ("data: [DONE]\n\n" if done else "")).encode() + + +class _CallCounter(CustomLogger): + pre_call: int = 0 + success: int = 0 + + async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None: + self.pre_call += 1 + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.success += 1 + + +async def _messages(model: str, api_base: str, stream: bool) -> object: + return await anthropic_messages( + model=model, api_base=api_base, api_key="sk-x", max_tokens=10, messages=list(USER_TURN), stream=stream + ) + + +async def _messages_to_chat(stream: bool) -> object: + return await _messages(CHAT_MODEL, CHAT_BASE, stream) + + +async def _messages_to_responses(stream: bool) -> object: + return await _messages("openai/gpt-5.6-luna", RESPONSES_BASE, stream) + + +async def _chat_to_responses(stream: bool) -> object: + return await litellm.acompletion( + model=RESPONSES_ONLY_MODEL, api_base=RESPONSES_BASE, api_key="sk-x", messages=list(USER_TURN), stream=stream + ) + + +async def _responses_to_chat(stream: bool) -> object: + return await litellm.aresponses( + model="deepseek/served-model", api_base=CHAT_BASE, api_key="sk-x", input="hi", stream=stream + ) + + +async def _drained(reply: object) -> None: + if isinstance(reply, AsyncIterable): + async for _ in cast(AsyncIterable[object], reply): # cast-ok: every bridged stream is drained the same way + pass + + +async def _settled(counter: _CallCounter) -> _CallCounter: + for _ in range(200): + if counter.success: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.1) + return counter + + +_BRIDGES: Final = ( + pytest.param(_messages_to_chat, "chat", id="messages_to_chat"), + pytest.param(_messages_to_responses, "responses", id="messages_to_responses"), + pytest.param(_chat_to_responses, "responses", id="chat_to_responses"), + pytest.param(_responses_to_chat, "chat", id="responses_to_chat"), +) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize(("bridge", "upstream_api"), _BRIDGES) +@pytest.mark.asyncio +async def test_a_bridged_request_runs_hooks_logs_and_calls_the_provider_once( + bridge: Callable[[bool], Coroutine[object, object, object]], + upstream_api: str, + stream: bool, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +): + counter: Final = _CallCounter() + _isolate_model_cost(monkeypatch) + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + litellm.register_model( + {RESPONSES_ONLY_MODEL: {"litellm_provider": "openai", "mode": "responses", "max_tokens": 100}} + ) + upstream: Final = ( + respx_mock.post(f"{CHAT_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=_sse(_CHAT_CHUNKS, done=True) + ) + if stream + else httpx.Response(200, json=_CHAT_REPLY) + ) + if upstream_api == "chat" + else respx_mock.post(f"{RESPONSES_BASE}/responses").mock( + return_value=httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=_sse(_RESPONSES_EVENTS, done=False) + ) + if stream + else httpx.Response(200, json=_RESPONSES_REPLY) + ) + ) + + await _drained(await bridge(stream)) + settled: Final = await _settled(counter) + + assert upstream.call_count == 1 + assert (settled.pre_call, settled.success) == (1, 1) diff --git a/tests/unit/proxy/common_utils/test_load_config_utils.py b/tests/unit/proxy/common_utils/test_load_config_utils.py index 927e6ee511d..107ea31c607 100644 --- a/tests/unit/proxy/common_utils/test_load_config_utils.py +++ b/tests/unit/proxy/common_utils/test_load_config_utils.py @@ -20,7 +20,7 @@ class TestGetFileContentsFromS3: """Test suite for S3 config loading functionality.""" @patch("boto3.client") - @patch("litellm.main.bedrock_converse_chat_completion") + @patch("litellm.native.chat_providers.handlers.bedrock_converse_chat_completion") @patch("yaml.safe_load") def test_get_file_contents_from_s3_no_temp_file_creation( self, mock_yaml_load, mock_bedrock, mock_boto3_client diff --git a/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py index 60c3e4c22e1..669a3ab4545 100644 --- a/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py @@ -19,6 +19,7 @@ import logging from collections.abc import Callable, Coroutine, Mapping from datetime import datetime from threading import Event +from functools import partial from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -28,7 +29,6 @@ import respx import litellm -from litellm.caching.caching import DualCache from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost from litellm.types.utils import StandardLoggingPayload @@ -59,6 +59,7 @@ def _attach_mock_success_dispatch(mock_logging_obj, async_success_fn): mock_logging_obj.dispatch_success_handlers = dispatch_success_handlers mock_logging_obj.async_success_handler = async_success_fn + mock_logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, mock_logging_obj) async def _wait_until(condition: Callable[[], bool]) -> None: @@ -593,6 +594,7 @@ class TestDeferredStreamingClosure: response args on logging_obj instead of calling the closure directly.""" mock_logging_obj = MagicMock() mock_logging_obj._on_deferred_stream_complete = MagicMock() + mock_logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, mock_logging_obj) resp = await litellm.acompletion( model="gpt-3.5-turbo", @@ -1388,6 +1390,7 @@ class TestResponsesIteratorDeferredLogging: logging_obj = MagicMock() logging_obj.dispatch_success_handlers = dispatch_success_handlers + logging_obj.hold_stream_completion = partial(Logging.hold_stream_completion, logging_obj) return logging_obj, recorded @pytest.mark.asyncio diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index ef198b487b9..77a83b68b56 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1,3 +1,4 @@ +import datetime import json from collections.abc import Mapping, Sequence from itertools import chain @@ -13,6 +14,7 @@ from pydantic import BaseModel import litellm from litellm import CustomLLM from litellm.constants import REDACTED_BY_LITELLM +from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, @@ -92,8 +94,8 @@ async def test_aresponses_forwards_timeout_to_acompletion(): and similar providers, with calls falling back to the provider SDK default (~600s for Anthropic). """ - with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: - mock_acompletion.return_value = ModelResponse( + with patch("litellm.native.chat_plan.aexecute_planned_chat", new_callable=AsyncMock) as mock_chat: + mock_chat.return_value = ModelResponse( id="id", created=0, model="anthropic/claude-sonnet-4-5", @@ -108,8 +110,8 @@ async def test_aresponses_forwards_timeout_to_acompletion(): api_key="sk-ant-fake", ) - assert mock_acompletion.call_count == 1 - forwarded_timeout = mock_acompletion.call_args.kwargs.get("timeout") + assert mock_chat.call_count == 1 + forwarded_timeout = mock_chat.call_args.args[0].named.get("timeout") assert forwarded_timeout == 42, ( f"timeout was not forwarded to acompletion (got {forwarded_timeout!r}); " "this means Router(timeout=N) silently fails for providers on the " @@ -1569,3 +1571,151 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search [("tool_result", "toolu_standup"), ("text", None)], ] assert prisma_client.db.queries == 1 +class _DeploymentHookCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: object) -> None: + self.calls += 1 + + +@pytest.mark.asyncio +async def test_bridged_responses_call_runs_deployment_hooks_once(monkeypatch: pytest.MonkeyPatch): + counter: Final = _DeploymentHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=False) as upstream: + messages_route: Final = upstream.post("http://anthropic.test/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_bridge", + "type": "message", + "role": "assistant", + "model": "claude-bridge", + "content": [{"type": "text", "text": "from upstream"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) + ) + response: Final = await litellm.aresponses( + model="anthropic/claude-bridge", input="hi", api_base="http://anthropic.test", api_key="sk-test" + ) + + assert response.output[0].content[0].text == "from upstream" + assert messages_route.call_count == 1 + assert counter.calls == 1 + + +@pytest.mark.asyncio +async def test_bridged_responses_call_is_billed_like_the_chat_call_for_one_hour_cache_writes_and_web_searches( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + usage: Final = { + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 2000, + "cache_read_input_tokens": 3000, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 2000}, + "server_tool_use": {"web_search_requests": 2}, + } + upstream_message: Final = { + "id": "msg_bridge_cost", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "searched"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": usage, + } + credentials: Final = {"api_base": "http://anthropic.test", "api_key": "sk-test"} + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("http://anthropic.test/v1/messages").mock(return_value=httpx.Response(200, json=upstream_message)) + chat: Final = await litellm.acompletion( + model="anthropic/claude-sonnet-4-5", messages=[{"role": "user", "content": "hi"}], **credentials + ) + bridged: Final = await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input="hi", **credentials) + + chat_cost: Final = chat._hidden_params["response_cost"] + assert isinstance(chat_cost, float) and chat_cost > 0 + assert bridged._hidden_params["response_cost"] == pytest.approx(chat_cost) + + +_ANTHROPIC_MESSAGE: Final = { + "id": "msg_bridge_headers", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, +} + + +@pytest.mark.asyncio +async def test_bridged_responses_call_names_the_provider_request_id_in_its_headers(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("http://anthropic.test/v1/messages").mock( + return_value=httpx.Response(200, json=_ANTHROPIC_MESSAGE, headers={"x-request-id": "req-upstream-1"}) + ) + bridged: Final = await litellm.aresponses( + model="anthropic/claude-sonnet-4-5", input="hi", api_base="http://anthropic.test", api_key="sk-test" + ) + + assert bridged._hidden_params["additional_headers"]["llm_provider-request-id"] == "req-upstream-1" + + +@pytest.mark.asyncio +async def test_bridged_responses_stream_opens_with_the_cost_breakdown_headers(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.common_request_processing import cost_headers + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + logging_obj: Final = Logging( + model="anthropic/claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="aresponses", + start_time=datetime.datetime.now(datetime.timezone.utc), + litellm_call_id="bridge-stream-headers-0000-4000-8000-000000000001", + function_id="1234", + ) + events: Final = ( + {"type": "message_start", "message": {**_ANTHROPIC_MESSAGE, "content": [], "stop_reason": None}}, + {"type": "message_stop"}, + ) + sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("http://anthropic.test/v1/messages").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=sse) + ) + stream: Final = await litellm.aresponses( + model="anthropic/claude-sonnet-4-5", + input="hi", + stream=True, + api_base="http://anthropic.test", + api_key="sk-test", + litellm_logging_obj=logging_obj, + ) + headers: Final = cost_headers(None, logging_obj) + async for _ in stream: + pass + + assert headers["x-litellm-response-cost-output"] == "0.0" + assert headers["x-litellm-response-cost-discount-amount"] == "0.0" + assert headers["x-litellm-response-cost-margin-amount"] == "0.0" diff --git a/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_bridge.py index 24db797f53b..58b2ace65e2 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_bridge.py @@ -23,16 +23,14 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools(): choices=[ litellm.utils.Choices( index=0, - message=litellm.utils.Message( - role="assistant", content="Test response" - ), + message=litellm.utils.Message(role="assistant", content="Test response"), finish_reason="stop", ) ], ) - with patch("litellm.acompletion", new_callable=AsyncMock) as mock_acompletion: - mock_acompletion.return_value = mock_response + with patch("litellm.native.chat_plan.aexecute_planned_chat", new_callable=AsyncMock) as mock_chat: + mock_chat.return_value = mock_response request_model = "gemini/gemini-2.5-flash" await litellm.aresponses( @@ -41,11 +39,10 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools(): tools=[{"type": "web_search_preview", "search_context_size": "low"}], ) - # Verify that acompletion was called - assert mock_acompletion.called + assert mock_chat.called - # Get the call arguments - call_args, call_kwargs = mock_acompletion.call_args + plan = mock_chat.call_args.args[0] + call_kwargs = {**plan.kwargs, **plan.named} # Verify the expected parameters were passed print( @@ -65,5 +62,5 @@ async def test_mock_basic_google_ai_studio_responses_api_with_tools(): call_kwargs["messages"][0]["content"] == "what is the latest version of supabase python package and when was it released?" ) - assert "tools" not in call_kwargs - assert "tool_choice" not in call_kwargs + assert call_kwargs.get("tools") is None + assert call_kwargs.get("tool_choice") is None diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..c64a2689f0b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -1,31 +1,24 @@ -"""Regression tests for the responses -> completion fallback bridge guard. - -When the Responses API falls back to chat completions (no native responses -config), it must tag the forwarded ``litellm.completion`` / ``litellm.acompletion`` -call with ``_skip_responses_api_bridge=True`` so ``completion()`` does not bridge -the request straight back to the Responses API and mutually recurse forever. - -Both fallback paths are covered: the sync ``response_api_handler`` (``_is_async`` -False) and the async ``async_response_api_handler`` (``_is_async`` True). The -module-level ``litellm.completion`` / ``litellm.acompletion`` are patched to -capture the forwarded kwargs; if the flag-setting line is removed the captured -kwargs lack the flag and these tests fail. -""" - +import asyncio import json +import math from collections.abc import Mapping, Sequence +from datetime import datetime from types import SimpleNamespace from typing import Final from unittest.mock import patch import httpx import pytest +import respx +from pydantic import TypeAdapter import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) +from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD @@ -35,50 +28,6 @@ class _StopForwarding(Exception): """Raised by the mocked (a)completion once the forwarded kwargs are captured.""" -def test_sync_fallback_tags_skip_responses_api_bridge(): - handler = LiteLLMCompletionTransformationHandler() - captured: dict = {} - - def fake_completion(**kwargs): - captured.update(kwargs) - raise _StopForwarding() - - with patch("litellm.completion", fake_completion): - with pytest.raises(_StopForwarding): - handler.response_api_handler( - model="gpt-4o", - input="hello", - responses_api_request={}, - custom_llm_provider="openai", - _is_async=False, - ) - - assert captured.get("_skip_responses_api_bridge") is True - - -@pytest.mark.asyncio -async def test_async_fallback_tags_skip_responses_api_bridge(): - handler = LiteLLMCompletionTransformationHandler() - captured: dict = {} - - async def fake_acompletion(**kwargs): - captured.update(kwargs) - raise _StopForwarding() - - with patch("litellm.acompletion", fake_acompletion): - coro = handler.response_api_handler( - model="gpt-4o", - input="hello", - responses_api_request={}, - custom_llm_provider="openai", - _is_async=True, - ) - with pytest.raises(_StopForwarding): - await coro - - assert captured.get("_skip_responses_api_bridge") is True - - _CODEX_ADDITIONAL_TOOLS_ITEM = { "type": "additional_tools", "id": "at_codex", @@ -303,10 +252,88 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ) assert provider.request_body is not None, "the bridged turn never reached the provider" - assert [ - (message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"] - ] == [ + assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [ ("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])), ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), ] + + +_LOGGED_ROW: Final = TypeAdapter(dict[str, object]) + + +class _SuccessRowRecorder(CustomLogger): + rows: tuple[dict[str, object], ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.rows = (*self.rows, _LOGGED_ROW.validate_python(kwargs["standard_logging_object"])) + + +@pytest.mark.asyncio +async def test_bridged_responses_stream_logs_one_responses_row_billed_from_provider_usage( + monkeypatch: pytest.MonkeyPatch, +): + recorder: Final = _SuccessRowRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, "cache_read_input_token_cost": 1e-7} + litellm.register_model( + {"deepseek/bridge-stream-model": {"litellm_provider": "deepseek", "mode": "chat", "max_tokens": 100, **prices}} + ) + chunk: Final[dict[str, object]] = {"id": "chatcmpl-stream", "object": "chat.completion.chunk", "created": 1} + chunks: Final[tuple[dict[str, object], ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + { + **chunk, + "choices": [], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + "prompt_tokens_details": {"cached_tokens": 40}, + }, + }, + ) + sse: Final = ("".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + "data: [DONE]\n\n").encode() + + with respx.mock(assert_all_called=False) as upstream: + chat_route: Final = upstream.post("https://example.invalid/v1/chat/completions").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse) + ) + stream: Final = await litellm.aresponses( + model="deepseek/bridge-stream-model", + api_base="https://example.invalid/v1", + api_key="x", + input="hi", + stream=True, + ) + assert isinstance(stream, ResponsesAPIStreamingIterator) + async for _ in stream: + pass + for _ in range(100): + if recorder.rows: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.1) + + assert stream.completed_response is not None + assert chat_route.call_count == 1 + assert len(recorder.rows) == 1 + row: Final = recorder.rows[0] + logged: Final = _LOGGED_ROW.validate_python(row["response"]) + assert logged["object"] == "response" + assert logged["status"] == "completed" + assert row["prompt_tokens"] == 100 + assert row["completion_tokens"] == 20 + cost: Final = row["response_cost"] + assert isinstance(cost, float) + assert math.isclose( + cost, + 60 * prices["input_cost_per_token"] + + 40 * prices["cache_read_input_token_cost"] + + 20 * prices["output_cost_per_token"], + ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_image_generation_output.py b/tests/unit/responses/litellm_completion_transformation/test_image_generation_output.py index 80e5335b4d4..dc4122a5c7f 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_image_generation_output.py +++ b/tests/unit/responses/litellm_completion_transformation/test_image_generation_output.py @@ -23,33 +23,20 @@ class TestExtractBase64FromDataUrl: def test_extracts_base64_from_data_url(self): """Should extract pure base64 from data URL with prefix""" data_url = "data:image/png;base64,iVBORw0KGgoAAAANS" - result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url( - data_url - ) + result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(data_url) assert result == "iVBORw0KGgoAAAANS" def test_returns_base64_as_is_if_no_prefix(self): """Should return base64 as-is if no data: prefix""" pure_base64 = "iVBORw0KGgoAAAANS" - result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url( - pure_base64 - ) + result = LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(pure_base64) assert result == pure_base64 def test_handles_invalid_inputs(self): """Should return None for empty/None/malformed inputs""" - assert ( - LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("") is None - ) - assert ( - LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(None) is None - ) - assert ( - LiteLLMCompletionResponsesConfig._extract_base64_from_data_url( - "data:image/png;base64" - ) - is None - ) + assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("") is None + assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url(None) is None + assert LiteLLMCompletionResponsesConfig._extract_base64_from_data_url("data:image/png;base64") is None class TestExtractImageGenerationOutputItems: @@ -75,10 +62,8 @@ class TestExtractImageGenerationOutputItems: mock_choice.message = mock_message mock_choice.finish_reason = "stop" - result = ( - LiteLLMCompletionResponsesConfig._extract_image_generation_output_items( - choice=mock_choice, - ) + result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items( + choice=mock_choice, ) assert len(result) == 2 @@ -99,10 +84,8 @@ class TestExtractImageGenerationOutputItems: mock_choice.message = mock_message mock_choice.finish_reason = "stop" - result = ( - LiteLLMCompletionResponsesConfig._extract_image_generation_output_items( - choice=mock_choice, - ) + result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items( + choice=mock_choice, ) assert result == [] @@ -122,10 +105,8 @@ class TestExtractImageGenerationOutputItems: mock_choice.message = mock_message mock_choice.finish_reason = "length" - result = ( - LiteLLMCompletionResponsesConfig._extract_image_generation_output_items( - choice=mock_choice, - ) + result = LiteLLMCompletionResponsesConfig._extract_image_generation_output_items( + choice=mock_choice, ) assert result[0].status == "incomplete" @@ -200,10 +181,7 @@ class TestImageGenerationOutputItemIds: def _choice_with_images(self, count): mock_message = Mock(spec=Message) - mock_message.images = [ - {"image_url": {"url": f"data:image/png;base64,IMG{idx}"}} - for idx in range(count) - ] + mock_message.images = [{"image_url": {"url": f"data:image/png;base64,IMG{idx}"}} for idx in range(count)] mock_choice = Mock(spec=Choices) mock_choice.message = mock_message mock_choice.finish_reason = "stop" diff --git a/tests/unit/responses/litellm_completion_transformation/test_reasoning_input_item_preservation.py b/tests/unit/responses/litellm_completion_transformation/test_reasoning_input_item_preservation.py index 5e001bdbbbb..cf3befe6e35 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_reasoning_input_item_preservation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_reasoning_input_item_preservation.py @@ -178,16 +178,12 @@ class TestEncryptedReasoningRoundTrip: item = { "type": "reasoning", "id": "rs_1", - "encrypted_content": json.dumps( - [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}] - ), + "encrypted_content": json.dumps([{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}]), } messages = _transform_item(item) assert len(messages) == 1 assert messages[0]["content"] is None - assert messages[0]["thinking_blocks"] == [ - {"type": "thinking", "thinking": "hidden", "signature": "sig-one"} - ] + assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}] def test_unsigned_blocks_dropped(self): """Blocks without a signature or redacted payload are not replayed.""" @@ -229,9 +225,7 @@ class TestEncryptedReasoningRoundTrip: ) assert len(messages) == 1 assert messages[0]["reasoning_content"] == "look it up" - assert messages[0]["thinking_blocks"] == [ - {"type": "thinking", "thinking": "hidden", "signature": "sig-one"} - ] + assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "hidden", "signature": "sig-one"}] assert len(messages[0]["tool_calls"]) == 1 def test_replayed_blocks_precede_existing_blocks(self): diff --git a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py index 093d1744418..d3229913cc0 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py +++ b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py @@ -58,5 +58,7 @@ def test_a_reasoning_item_is_litellm_minted_by_its_id_or_by_its_encoded_thinking "encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB, } ) - assert not is_litellm_minted_reasoning_item({"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"}) + assert not is_litellm_minted_reasoning_item( + {"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"} + ) assert not is_litellm_minted_reasoning_item("a bare string input") diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 8d5befd9443..6867f5ed739 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1245,3 +1245,134 @@ async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_ assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"] assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"] + + +def _text(content: object) -> object: + if isinstance(content, list): + return "".join(part["text"] for part in content) + return content + + +def _responses_spend_log(request_id: str, session_id: str, prompt: str, output: list[dict[str, object]]) -> dict: + return { + "request_id": request_id, + "call_type": "aresponses", + "session_id": session_id, + "proxy_server_request": {"input": [{"role": "user", "content": prompt}], "model": "claude-bridge"}, + "response": {"id": request_id, "object": "response", "status": "completed", "output": output}, + } + + +@pytest.mark.asyncio +async def test_message_history_reads_responses_shaped_rows_next_to_chat_shaped_rows(): + session_id = "0f6f2d8e-6c8a-4b55-9a39-3d0b1f3c9e11" + fake_prisma_client = _FakePrismaClient( + results=[ + [ + _spend_log("chatcmpl-3333", session_id, "My favorite color is chartreuse.", "Got it."), + _responses_spend_log( + "resp_4444", + session_id, + "What is the weather in Lisbon?", + [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Let me check.", "annotations": []}], + }, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_weather", + "name": "get_weather", + "arguments": '{"city": "Lisbon"}', + }, + ], + ), + ] + ] + ) + + with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id("resp_4444") + + messages: Final = result["messages"] + assert [(message.get("role"), _text(message.get("content"))) for message in messages[:4]] == [ + ("user", "My favorite color is chartreuse."), + ("assistant", "Got it."), + ("user", "What is the weather in Lisbon?"), + ("assistant", "Let me check."), + ] + tool_calls: Final = [call for message in messages[3:] for call in message.get("tool_calls") or []] + assert [(call["id"], call["function"]["name"], call["function"]["arguments"]) for call in tool_calls] == [ + ("call_weather", "get_weather", '{"city": "Lisbon"}') + ] + + +_WEB_SEARCH_RESULT: Final = { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [{"type": "web_search_result", "url": "https://x.com", "title": "X", "encrypted_content": "EC"}], +} + + +def _bridged_anthropic_web_search_turn() -> dict[str, object]: + """A Claude turn served through the Responses to chat bridge, stored as its Responses object.""" + return { + "id": "resp_1", + "object": "response", + "output": [ + { + "id": "ws_srvtoolu_1", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "query": "weather sf", + "sources": [{"type": "url", "url": "https://x.com"}], + }, + }, + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}], + }, + ], + "provider_specific_fields": {"web_search_results": [_WEB_SEARCH_RESULT]}, + } + + +def test_message_history_replays_a_bridged_anthropic_web_search_as_server_tool_blocks(): + from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt + + history: Final = ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( + spend_log={"request_id": "resp_1", "response": _bridged_anthropic_web_search_turn()}, + proxy_server_request_dict={"input": "weather in sf?"}, + chat_completion_message_history=[], + ) + anthropic_messages: Final = anthropic_messages_pt( + messages=history, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assert [message["role"] for message in anthropic_messages] == ["user", "assistant"] + assistant_content: Final = anthropic_messages[1]["content"] + assert [block["type"] for block in assistant_content] == ["text", "server_tool_use", "web_search_tool_result"] + assert assistant_content[1]["id"] == "srvtoolu_1" + assert assistant_content[1]["input"] == {"query": "weather sf"} + assert assistant_content[2]["content"] == _WEB_SEARCH_RESULT["content"] + + +def test_message_history_keeps_a_web_search_without_a_stored_result_as_text(): + turn: Final = {**_bridged_anthropic_web_search_turn(), "provider_specific_fields": None} + + history: Final = ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( + spend_log={"request_id": "resp_1", "response": turn}, + proxy_server_request_dict=None, + chat_completion_message_history=[], + ) + + assert str(_text_of(history[0]["content"])).startswith("Hosted web search: ") diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py index 1565820cbbd..c19b96a35ed 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler_with_cold_storage.py @@ -78,9 +78,7 @@ class TestColdStorageObjectKeyIntegration: metadata = {"user_api_key": "test_key", "user_api_key_team_id": "test_team"} # Call the function - result = _get_spend_logs_metadata( - metadata=metadata, cold_storage_object_key="test/path/to/object.json" - ) + result = _get_spend_logs_metadata(metadata=metadata, cold_storage_object_key="test/path/to/object.json") # Verify the object key is stored in the result assert result.get("cold_storage_object_key") == "test/path/to/object.json" @@ -104,11 +102,7 @@ class TestColdStorageObjectKeyIntegration: } # Test the extraction method - object_key = ( - ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log( - spend_log - ) - ) + object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) assert object_key == "test/path/to/object.json" @@ -128,11 +122,7 @@ class TestColdStorageObjectKeyIntegration: } # Test the extraction method - object_key = ( - ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log( - spend_log - ) - ) + object_key = ResponsesSessionHandler._get_cold_storage_object_key_from_spend_log(spend_log) assert object_key == "test/path/to/object.json" @@ -148,20 +138,17 @@ class TestColdStorageObjectKeyIntegration: # Mock the custom logger mock_logger = AsyncMock() - mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = ( - AsyncMock(return_value={"test": "data"}) + mock_logger.get_proxy_server_request_from_cold_storage_with_object_key = AsyncMock( + return_value={"test": "data"} ) with ( - patch.object( - handler, "_select_custom_logger_for_cold_storage", return_value="s3_v2" - ), + patch.object(handler, "_select_custom_logger_for_cold_storage", return_value="s3_v2"), patch( "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name", return_value=mock_logger, ), ): - result = await handler.get_proxy_server_request_from_cold_storage_with_object_key( object_key="test/path/to/object.json" ) @@ -184,9 +171,7 @@ class TestColdStorageObjectKeyIntegration: s3_logger = S3Logger(s3_bucket_name="test-bucket") # Mock the _download_object_from_s3 method - with patch.object( - s3_logger, "_download_object_from_s3", return_value={"test": "data"} - ) as mock_download: + with patch.object(s3_logger, "_download_object_from_s3", return_value={"test": "data"}) as mock_download: result = await s3_logger.get_proxy_server_request_from_cold_storage_with_object_key( object_key="test/path/to/object.json" ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 86773dc7ddf..b7b575d06fe 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -38,9 +38,7 @@ from litellm.types.utils import ( ) CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256" -RESPONSE_ID_EVENT_TYPES = frozenset( - {"response.created", "response.in_progress", "response.completed"} -) +RESPONSE_ID_EVENT_TYPES = frozenset({"response.created", "response.in_progress", "response.completed"}) def _chunk(content: str, finish_reason: str | None = None) -> ModelResponseStream: @@ -93,11 +91,7 @@ def _build_iterator(chunks) -> LiteLLMCompletionStreamingIterator: def _response_ids(events) -> list[str]: - return [ - event.response.id - for event in events - if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES - ] + return [event.response.id for event in events if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] def test_tool_call_delta_is_emitted_as_responses_events(): @@ -228,9 +222,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to content=None, provider_specific_fields={ "web_search_results": [first_result], - "web_search_calls": [ - build_web_search_call(call_ids[0], {"query": "one"}, first_result) - ] + "web_search_calls": [build_web_search_call(call_ids[0], {"query": "one"}, first_result)] if tool_type != "function" and first_result["type"] == "web_search_tool_result" else [], }, @@ -329,9 +321,7 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to assert {event["output_index"] for event in search_events} == {completed_indexes[item["id"]]} assert search_events[0]["item"]["status"] == "in_progress" assert search_events[-1]["item"] == item - assert item["status"] == ( - "failed" if result_kind == "error" and call_id.endswith("01Search") else "completed" - ) + assert item["status"] == ("failed" if result_kind == "error" and call_id.endswith("01Search") else "completed") assert item["action"]["type"] == "search" assert item["action"]["query"] == ("one" if call_id.endswith("01Search") else "two") assert item["action"]["queries"] == [item["action"]["query"]] @@ -440,9 +430,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): ) # Create a chunk with a large arguments string that should be split - large_arguments = ( - '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars - ) + large_arguments = '{"param1": "value1", "param2": "value2", "param3": "value3"}' # 67 chars chunk = ModelResponseStream( id="chunk-1", created=123, @@ -497,9 +485,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # Keep draining pending events (expected: ceil(67 / 10) = 7 delta events) while iterator._pending_tool_events: - evt = iterator._transform_chat_completion_chunk_to_response_api_chunk( - empty_chunk - ) + evt = iterator._transform_chat_completion_chunk_to_response_api_chunk(empty_chunk) if evt and evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA: delta_events.append(evt) @@ -552,20 +538,12 @@ def test_tool_call_delta_without_id_uses_index_mapping(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] streamed_arguments = "".join(evt.delta for evt in delta_events) assert streamed_arguments == '{"location": "New York"}' - output_item_added_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - ] + output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] assert len(output_item_added_events) == 1 assert output_item_added_events[0].item.id == "fc_call_abc123" assert output_item_added_events[0].item.call_id == "call_abc123" @@ -606,18 +584,10 @@ def test_parallel_tool_calls_without_ids_use_index_mapping(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - output_item_added_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - ] + output_item_added_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] assert len(output_item_added_events) == 2 - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] arguments_by_call_id = {} for evt in delta_events: arguments_by_call_id.setdefault(evt.item_id, "") @@ -670,11 +640,7 @@ def test_reused_index_with_new_call_id_marks_fallback_ambiguous(): while iterator._pending_tool_events: all_events.append(iterator._pending_tool_events.pop(0)) - delta_events = [ - evt - for evt in all_events - if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA - ] + delta_events = [evt for evt in all_events if evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA] arguments_by_call_id = {} for evt in delta_events: arguments_by_call_id.setdefault(evt.item_id, "") @@ -741,22 +707,17 @@ def test_sync_streaming_events_share_the_chat_completion_response_id(): assert len(response_ids) == 3 assert len(set(response_ids)) == 1 assert ( - ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] - == CHAT_COMPLETION_ID + ResponsesAPIRequestUtils.decode_responses_api_response_id(response_ids[0])["response_id"] == CHAT_COMPLETION_ID ) @pytest.mark.asyncio async def test_streaming_emits_every_chunk_after_priming_the_response_id(): - iterator = _build_iterator( - [_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")] - ) + iterator = _build_iterator([_chunk("Hel"), _chunk("lo"), _chunk("!", finish_reason="stop")]) events = [event async for event in iterator] - deltas = "".join( - event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta" - ) + deltas = "".join(event.delta for event in events if getattr(event, "type", None) == "response.output_text.delta") assert deltas == "Hello!" @@ -802,7 +763,9 @@ async def test_leading_empty_choices_chunk_does_not_kill_the_stream(): event_types = [getattr(event, "type", None) for event in events] assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1 - assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + assert ( + "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + ) assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED @@ -986,9 +949,7 @@ def test_streamed_unrecognized_tool_choice_is_echoed_as_auto() -> None: litellm_metadata={}, ) - response_events: Final = [ - event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES - ] + response_events: Final = [event for event in iterator if getattr(event, "type", None) in RESPONSE_ID_EVENT_TYPES] assert [event.response.tool_choice for event in response_events] == ["auto", "auto", "auto"] @@ -1038,9 +999,7 @@ def _signed_thinking_and_text_in_one_chunk() -> tuple[ModelResponseStream, ...]: chunk: Final = _chunk("Hello!", finish_reason="stop") signed_block: Final = {"type": "thinking", "thinking": "", "signature": "sig_only"} delta: Final = Delta(**{**chunk.choices[0].delta.model_dump(), "thinking_blocks": [signed_block]}) - return ( - chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}), - ) + return (chunk.model_copy(update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason="stop")]}),) StreamChunks = Callable[[], tuple[ModelResponseStream, ...]] @@ -1560,9 +1519,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ model="claude-sonnet-5-5", object="chat.completion.chunk", choices=[ - StreamingChoices( - index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None - ) + StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None) ], ) for index, delta in enumerate(deltas) diff --git a/tests/unit/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py b/tests/unit/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py index 3dec1d571c4..3853a47422b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py +++ b/tests/unit/responses/litellm_completion_transformation/test_tool_output_order_preserved_for_gemini.py @@ -60,11 +60,7 @@ def test_function_call_output_stays_adjacent_to_tool_call(): assistant_ok_idx = i elif isinstance(content, list): for block in content: - if ( - isinstance(block, dict) - and block.get("type") == "text" - and block.get("text") == "Done." - ): + if isinstance(block, dict) and block.get("type") == "text" and block.get("text") == "Done.": assistant_ok_idx = i break diff --git a/tests/unit/responses/test_bridge_executor.py b/tests/unit/responses/test_bridge_executor.py new file mode 100644 index 00000000000..a7708f2f866 --- /dev/null +++ b/tests/unit/responses/test_bridge_executor.py @@ -0,0 +1,87 @@ +import json +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.responses.bridge_executor import native_logging_obj + +API_BASE: Final = "https://bridge-executor.invalid/v1" +_CHAT_REPLY: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "m", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, +} + + +@pytest.fixture(autouse=True) +def _offline(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.parametrize( + ("surface", "litellm_params", "model_info", "upstream_model"), + [ + ("messages", {"model": "openai/gpt-4.1", "use_chat_completions_api": True}, None, "gpt-4.1"), + ("messages", {"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}, "gpt-4.1"), + ("messages", {"model": "openai/chat_completions/o3-pro"}, None, "o3-pro"), + ("responses", {"model": "openai/chat_completions/o3-pro"}, None, "o3-pro"), + ], +) +@pytest.mark.asyncio +async def test_a_deployment_asking_for_chat_completions_is_sent_to_chat_without_the_prefix( + surface: str, + litellm_params: dict[str, object], + model_info: dict[str, object] | None, + upstream_model: str, + respx_mock: respx.MockRouter, +): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "m", + "litellm_params": {"api_base": API_BASE, "api_key": "k", **litellm_params}, + **({"model_info": model_info} if model_info is not None else {}), + } + ], + num_retries=0, + ) + + if surface == "messages": + await router.aanthropic_messages(model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}]) + else: + await router.aresponses(model="m", input="hi") + + assert chat.call_count == 1 + assert json.loads(chat.calls.last.request.content)["model"] == upstream_model + + +@pytest.mark.parametrize(("response_cache", "native"), [(None, True), (litellm.Cache(type="local"), False)]) +def test_a_bridged_responses_call_goes_through_aresponses_when_the_response_cache_is_on( + response_cache: litellm.Cache | None, native: bool, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cache", response_cache) + logging_obj: Final = Logging( + model="gpt-4.1", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="bridge-executor-cache", + function_id="bridge-executor-cache", + ) + + chosen: Final = native_logging_obj(logging_obj, {"api_key": "k"}, {"model": "openai/gpt-4.1", "input": "hi"}) + + assert (chosen is logging_obj) is native diff --git a/tests/unit/responses/test_responses_supported_endpoints_passthrough.py b/tests/unit/responses/test_responses_supported_endpoints_passthrough.py index 4ff4963423e..a0bb774a1f6 100644 --- a/tests/unit/responses/test_responses_supported_endpoints_passthrough.py +++ b/tests/unit/responses/test_responses_supported_endpoints_passthrough.py @@ -15,7 +15,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig -from litellm.responses.main import _resolve_responses_api_provider_config +from litellm.responses.main import resolve_responses_api_provider_config from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse @@ -115,13 +115,13 @@ def _respx_interceptable_httpx_client(monkeypatch): ], ) def test_resolver_opt_in_gates_openai_like_config(model_info, expected_type): - config = _resolve_responses_api_provider_config("my-model", "custom_openai", model_info, None) + config = resolve_responses_api_provider_config("my-model", "custom_openai", model_info, None) assert type(config) is expected_type def test_resolver_keeps_native_provider_config(): """`openai/` already routes /v1/responses natively; the opt-in must not swap its config.""" - config = _resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN, None) + config = resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN, None) assert type(config) is OpenAIResponsesAPIConfig diff --git a/tests/unit/test_check_native_imports.py b/tests/unit/test_check_native_imports.py new file mode 100644 index 00000000000..2d2cf3b6299 --- /dev/null +++ b/tests/unit/test_check_native_imports.py @@ -0,0 +1,68 @@ +import subprocess +import sys +from pathlib import Path +from typing import Final + +import pytest + +from scripts.check_native_imports import main, violations + +CHECKER: Final = Path(__file__).resolve().parents[2] / "scripts/check_native_imports.py" + + +@pytest.mark.parametrize( + "path, source", + ( + ("litellm/native/chat.py", "from litellm.main import execute_chat"), + ("litellm/native/chat.py", "from litellm import main"), + ("litellm/native/chat.py", "import litellm.main"), + ("litellm/native/chat.py", "import litellm.main as legacy"), + ("litellm/native/chat.py", "from ..main import completion"), + ("litellm/native/chat.py", "from .. import main"), + ("litellm/native/chat_providers/azure.py", "from ...main import completion"), + ("litellm/native/chat.py", "import litellm\n\ndef run():\n return litellm.main.completion()"), + ), +) +def test_native_module_importing_main_is_rejected(path, source): + assert violations(Path(path), source) + + +@pytest.mark.parametrize( + "path, source", + ( + ("litellm/native/chat.py", "from litellm.native.chat_providers.azure import complete_azure"), + ("litellm/native/chat.py", "import litellm\n\ndef run():\n return litellm.completion"), + ("litellm/native/chat.py", "from litellm.maintenance import x"), + ("litellm/native/chat.py", "from .chat_mock import mock_completion"), + ("litellm/native/chat_providers/azure.py", "from ..chat_mock import mock_completion"), + ("litellm/native/chat.py", "from litellm import utils"), + ), +) +def test_native_module_without_main_dependency_passes(path, source): + assert violations(Path(path), source) == () + + +def test_command_reports_the_offending_line_and_passes_once_it_is_gone(tmp_path, monkeypatch, capsys): + package: Final = tmp_path / "litellm/native/chat_providers" + package.mkdir(parents=True) + module: Final = package / "azure.py" + module.write_text("import os\nfrom litellm.main import completion\n") + monkeypatch.chdir(tmp_path) + assert main() == 1 + assert "litellm/native/chat_providers/azure.py:2:" in capsys.readouterr().err + rejected: Final = subprocess.run( + [sys.executable, "-I", str(CHECKER)], cwd=tmp_path, capture_output=True, text=True, check=False + ) + assert rejected.returncode == 1 + + module.write_text("import os\n") + assert main() == 0 + accepted: Final = subprocess.run( + [sys.executable, "-I", str(CHECKER)], cwd=tmp_path, capture_output=True, text=True, check=False + ) + assert accepted.returncode == 0 + + +def test_the_native_package_has_no_main_dependency(monkeypatch): + monkeypatch.chdir(CHECKER.parents[1]) + assert main() == 0 diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 571f890a4e6..cda1e732ad7 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -7,6 +7,7 @@ import json import logging import os import threading +import time import urllib.parse from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor @@ -20,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError, AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, OpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion from pydantic import BaseModel, ConfigDict, TypeAdapter @@ -45,10 +46,12 @@ from litellm.litellm_core_utils.get_litellm_params import stored_control_options from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.native import chat as native_chat from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import ( + CallTypes, Delta, ModelResponseStream, StandardCallbackDynamicParams, @@ -517,7 +520,7 @@ def test_drop_anthropic_only_tool_keys_strips_tool_and_function_levels(key: str) "opaque_tool", ] - cleaned = litellm_main._drop_anthropic_only_tool_keys(tools=tools) + cleaned = native_chat._drop_anthropic_only_tool_keys(tools=tools) assert cleaned == [ {"type": "function", "name": "example_tool", "function": {"name": "example_tool"}}, @@ -553,6 +556,98 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx assert sent_tool["function"]["name"] == "write_file" +class _SyncSuccessCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.calls: list[str] = [] # mutable-ok: test recorder of success-callback events + + def log_success_event(self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object): + self.calls.append(str(kwargs.get("call_type"))) + + +def _settled_success_calls(counter: _SyncSuccessCounter, expected: int) -> int: + for _ in range(50): + if len(counter.calls) >= expected: + break + time.sleep(0.02) + return len(counter.calls) + + +class _DeploymentHookCounter(CustomLogger): + call_types: tuple[str, ...] = () + + async def async_pre_call_deployment_hook(self, kwargs: dict[str, object], call_type: CallTypes | None) -> None: + self.call_types = (*self.call_types, str(call_type)) + + +@pytest.mark.asyncio +async def test_responses_mode_model_without_a_responses_api_stays_on_chat( + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], monkeypatch: pytest.MonkeyPatch +): + counter: Final = _DeploymentHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + litellm.register_model( + {"groq/responses-only-model": {"litellm_provider": "groq", "mode": "responses", "max_tokens": 100}} + ) + api_base: Final = "https://example.invalid/openai/v1" + chat_route: Final = respx_mock.post(f"{api_base}/chat/completions").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + + response: Final = await litellm.acompletion( + model="groq/responses-only-model", + api_base=api_base, + api_key="sk-x", + messages=[{"role": "user", "content": "hi"}], + ) + + assert isinstance(response, litellm.ModelResponse) + assert response.id == openai_api_response["id"] + assert chat_route.call_count == 1 + assert counter.call_types == ("CallTypes.acompletion",) + + +def test_execute_chat_calls_the_provider_without_logging( + respx_mock: respx.MockRouter, openai_api_response, monkeypatch: pytest.MonkeyPatch +): + counter: Final = _SyncSuccessCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + api_base: Final = "http://localhost:12347/v1" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + messages: Final = [{"role": "user", "content": "hi"}] + logging_obj: Final = LiteLLMLogging( + model="hosted_vllm/served-model", + messages=messages, + stream=False, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="execute-chat-call", + function_id="execute-chat-fn", + ) + + response: Final = litellm_main.execute_chat( + model="hosted_vllm/served-model", + messages=list(messages), + kwargs={"api_base": api_base, "api_key": "sk-x", "litellm_logging_obj": logging_obj}, + non_default_params={}, + control_options=ControlOptions(), + prompt_id=None, + prompt_variables=None, + ) + + assert isinstance(response, litellm.ModelResponse) + assert response.id == openai_api_response["id"] + assert mock_route.call_count == 1 + assert _settled_success_calls(counter, expected=1) == 0 + + litellm.completion(model="hosted_vllm/served-model", messages=messages, api_base=api_base, api_key="sk-x") + assert _settled_success_calls(counter, expected=1) == 1 + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", @@ -2591,7 +2686,7 @@ def test_completion_optional_params_base_model( Regression for #29618: overwriting ``model`` with a friendly ``base_model`` label made Bedrock drop ``tools``/``tool_choice`` under ``drop_params``.""" - with patch("litellm.main.get_optional_params") as mock_get_optional_params: + with patch("litellm.native.chat.get_optional_params") as mock_get_optional_params: mock_get_optional_params.return_value = MagicMock() import litellm @@ -5684,7 +5779,7 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_ def test_aiohttp_openai_warns_only_when_http2_enabled( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool ): - from litellm.main import base_llm_aiohttp_handler + from litellm.native.chat_providers.handlers import base_llm_aiohttp_handler monkeypatch.setattr(litellm, "http2", http2_on) monkeypatch.delenv("LITELLM_HTTP2", raising=False) @@ -5985,7 +6080,6 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None: def test_acompletion_params(): import inspect - from litellm.types.completion import CompletionRequest acompletion_params_odict = inspect.signature(acompletion).parameters completion_params_dict = inspect.signature(completion).parameters diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 35fbd28eae7..fa4fc18185f 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -20,7 +20,7 @@ import pytest import litellm from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map -from litellm.main import _build_custom_pricing_entry +from litellm.native.request_support import _build_custom_pricing_entry from litellm.utils import _invalidate_model_cost_lowercase_map @@ -1677,7 +1677,7 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request + from litellm.native.request_support import register_custom_pricing_for_request deployment_id: Final = "baseten/zai-org/glm-5.2" builtin_key: Final = "baseten/zai-org/GLM-5.2" @@ -1686,7 +1686,7 @@ def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_prov builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) try: - _register_custom_pricing_for_request( + register_custom_pricing_for_request( model="zai-org/GLM-5.2", custom_llm_provider="openai", kwargs={ diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a0a7dd78193..6634886bd42 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1843,7 +1843,9 @@ def test_arouter_responses_api_bridge(): ## CONFIRM BRIDGE IS CALLED # an empty stream: the Router reads the first chunk before returning, and a stream of mocks never produces one - with patch.object(litellm, "responses", return_value=iter([])) as mock_responses: + from litellm.responses import bridge_executor + + with patch.object(bridge_executor, "native_responses", return_value=iter([])) as mock_responses: result = router.completion( model="[IP-approved] o3-pro", messages=[{"role": "user", "content": "Hello, world!"}],